From 19a34951cfc7e88584ce0530aee9c680172a4ad9 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 01:31:27 -0700 Subject: [PATCH 01/18] chore(tooling): align lint hooks and preserve copyright notices --- .pre-commit-config.yaml | 3 +-- pyproject.toml | 3 +++ scripts/add_license_header.py | 7 ++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d2a6fdfb7..64aa89f1e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,9 +12,8 @@ repos: - id: check-merge-conflict - id: debug-statements - # TODO: sync rev with ruff version in pyproject.toml (currently 0.15.8) - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.3 + rev: v0.15.8 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/pyproject.toml b/pyproject.toml index a06905cb4..ae6a31f5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -213,6 +213,9 @@ ignore = [ "C901", # too complex ] +[tool.ruff.lint.isort] +combine-as-imports = true + [tool.ruff.format] quote-style = "double" indent-style = "space" diff --git a/scripts/add_license_header.py b/scripts/add_license_header.py index 9bf8f787c..9451b08a5 100644 --- a/scripts/add_license_header.py +++ b/scripts/add_license_header.py @@ -66,10 +66,11 @@ def add_header_to_file(filepath: Path) -> bool: lines[i] = ( f"# SPDX-FileCopyrightText: Copyright (c) {YEAR} NVIDIA CORPORATION & AFFILIATES. All rights reserved." ) + updated = "\n".join(lines) + if updated != content: + filepath.write_text(updated) + return True break - content = "\n".join(lines) - filepath.write_text(content) - return True return False # Handle shebang From d286fe1ddefc42e8187479dfac4caa7996da914b Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 01:32:12 -0700 Subject: [PATCH 02/18] refactor(metrics): normalize steady-state verdicts with enums --- .../services/metrics_aggregator/snapshot.py | 3 +- src/inference_endpoint/metrics/report.py | 7 +- .../metrics/steady_state_diagnostics.py | 82 ++++++++++++++----- .../metrics_aggregator/test_snapshot.py | 52 ++++++++++-- tests/unit/metrics/test_report_builder.py | 20 +++++ .../metrics/test_steady_state_diagnostics.py | 37 +++++---- 6 files changed, 153 insertions(+), 48 deletions(-) diff --git a/src/inference_endpoint/async_utils/services/metrics_aggregator/snapshot.py b/src/inference_endpoint/async_utils/services/metrics_aggregator/snapshot.py index f32c59033..fe1b6d0b2 100644 --- a/src/inference_endpoint/async_utils/services/metrics_aggregator/snapshot.py +++ b/src/inference_endpoint/async_utils/services/metrics_aggregator/snapshot.py @@ -30,11 +30,12 @@ import msgspec import msgspec.msgpack + from inference_endpoint.core.record import TOPIC_FRAME_SIZE from inference_endpoint.metrics.steady_state_diagnostics import SteadyState -class SessionState(str, Enum): +class SessionState(str, Enum): # noqa: UP042 - repository wire-enum convention """The aggregator's session state at the time a snapshot was emitted. INITIALIZE → aggregator has been constructed but no ``STARTED`` event diff --git a/src/inference_endpoint/metrics/report.py b/src/inference_endpoint/metrics/report.py index 98834de34..731888461 100644 --- a/src/inference_endpoint/metrics/report.py +++ b/src/inference_endpoint/metrics/report.py @@ -36,7 +36,10 @@ from inference_endpoint.evaluation.accuracy_results import ( samples_weighted_average_accuracy, ) -from inference_endpoint.metrics.steady_state_diagnostics import SteadyState +from inference_endpoint.metrics.steady_state_diagnostics import ( + SteadyState, + normalize_steady_state_payload, +) from inference_endpoint.utils.version import get_version_info from ..utils import monotime_to_datetime @@ -193,7 +196,7 @@ def _steady_state_of(snap: dict) -> SteadyState | None: diagnostic. Dropping it makes the absence visible. Raising would suppress the report, and coercing could print a plausible wrong number. """ - raw = snap.get("steady_state") + raw = normalize_steady_state_payload(snap.get("steady_state")) if raw is None: return None try: diff --git a/src/inference_endpoint/metrics/steady_state_diagnostics.py b/src/inference_endpoint/metrics/steady_state_diagnostics.py index bfad3b8c4..83d53f7ba 100644 --- a/src/inference_endpoint/metrics/steady_state_diagnostics.py +++ b/src/inference_endpoint/metrics/steady_state_diagnostics.py @@ -61,8 +61,9 @@ from array import array from collections.abc import Callable, Sequence from dataclasses import dataclass, field +from enum import Enum from statistics import NormalDist, median, pstdev -from typing import Final, Literal, NamedTuple, TypedDict +from typing import Final, NamedTuple, TypedDict import msgspec import yaml @@ -134,8 +135,40 @@ # k* rises for noisy metrics. A higher fixed floor would only penalize clean runs. MIN_DUR_KSTAR_FLOOR = MIN_TREND_N + # A per-super-pass metric trajectory is classified into one of these states. -Verdict = Literal["up", "down", "steady", "insufficient"] +class Verdict(str, Enum): # noqa: UP042 - repository wire-enum convention + DRIFTING_UP = "drifting_up" + DRIFTING_DOWN = "drifting_down" + PLATEAU = "plateau" + INSUFFICIENT = "insufficient" + + +_LEGACY_TREND_VERDICTS = { + "up": Verdict.DRIFTING_UP, + "down": Verdict.DRIFTING_DOWN, + "steady": Verdict.PLATEAU, +} + + +def normalize_steady_state_payload(raw: object) -> object: + """Return a copy with supported trend aliases normalized to canonical enum values.""" + if not isinstance(raw, dict): + return raw + global_trend = raw.get("global_trend") + if not isinstance(global_trend, dict): + return raw + normalized = { + key: _LEGACY_TREND_VERDICTS.get(value, value) + if isinstance(value, str) + else value + for key, value in global_trend.items() + } + if normalized == global_trend: + return raw + result = dict(raw) + result["global_trend"] = normalized + return result # The verdict rides ``MetricsSnapshot`` and lands in ``result_summary.json``. @@ -1115,8 +1148,7 @@ def system_tps(out_tokens: int, elapsed_ns: int) -> float: # --------------------------------------------------------------------------- # -# Trend algorithms -- each returns a TrendResult with verdict in -# {"up", "steady", "down", "insufficient"}. +# Trend algorithms -- each returns a TrendResult with a typed verdict. # --------------------------------------------------------------------------- # @dataclass(frozen=True, slots=True) class TrendResult: @@ -1132,21 +1164,25 @@ class TrendResult: def _insufficient() -> TrendResult: - return TrendResult("insufficient") + return TrendResult(Verdict.INSUFFICIENT) def _direction(x: float) -> Verdict: - return "up" if x > 0 else "down" if x < 0 else "steady" + if x > 0: + return Verdict.DRIFTING_UP + if x < 0: + return Verdict.DRIFTING_DOWN + return Verdict.PLATEAU def _significant_verdict(effect: float, pvalue: float, alpha: float) -> Verdict: """up/down when the effect is significant (pvalue < alpha) in that direction.""" if pvalue < alpha: if effect > 0: - return "up" + return Verdict.DRIFTING_UP if effect < 0: - return "down" - return "steady" + return Verdict.DRIFTING_DOWN + return Verdict.PLATEAU def _median_or_floor(values: Sequence[float]) -> float: @@ -1259,7 +1295,7 @@ def theil_sen( ] slope = median(slopes) rel = slope * (n - 1) / _median_or_floor(values) - verdict: Verdict = "steady" if abs(rel) < rel_threshold else _direction(rel) + verdict = Verdict.PLATEAU if abs(rel) < rel_threshold else _direction(rel) return TrendResult(verdict, slope=slope, rel_drift=rel) @@ -1297,7 +1333,7 @@ def newey_west( return _insufficient() slope, sxx, resid = _ols(values) if sxx == 0: - return TrendResult("steady", slope=0.0) + return TrendResult(Verdict.PLATEAU, slope=0.0) xbar = (n - 1) / 2.0 u = [(x - xbar) * resid[x] for x in range(n)] if lag is None: @@ -1331,7 +1367,7 @@ def slope_vs_scatter( rel_drift = total_change / _median_or_floor(values) snr = abs(total_change) / (resid_std + ZERO_MEDIAN_FLOOR) drifting = abs(rel_drift) >= rel_threshold and snr >= snr_threshold - verdict: Verdict = _direction(rel_drift) if drifting else "steady" + verdict = _direction(rel_drift) if drifting else Verdict.PLATEAU return TrendResult(verdict, slope=slope, snr=snr, rel_drift=rel_drift) @@ -1353,7 +1389,7 @@ def cov_pass_row( # CoV is undefined with fewer than two points. Report inconclusive, not # PASS, so a short or empty window cannot masquerade as steady. if len(values) < 2: - return {b: None for b in bounds} + return dict.fromkeys(bounds) c = cov(values) return {b: c <= b for b in bounds} @@ -1420,7 +1456,7 @@ def _window_gate( if traj is None or len(traj) < MIN_TREND_N: return _GATE_SHORT if ( - gate(traj).verdict != "steady" + gate(traj).verdict is not Verdict.PLATEAU and abs(_rel_drift(traj)) >= TREND_REL_DRIFT_MIN ): # Break only on trends that are significant and practically large. @@ -1469,7 +1505,7 @@ def gated_trend_drifters( if traj is None or len(traj) < MIN_TREND_N: return None if ( - gate(traj).verdict != "steady" + gate(traj).verdict is not Verdict.PLATEAU and abs(_rel_drift(traj)) >= TREND_REL_DRIFT_MIN ): drifting.append(m.key) @@ -1580,7 +1616,9 @@ def global_trend( traj = super_pass_percentile_series( series[from_idx:], m.source_attr, m.percentile ) - out[m.key] = gate(traj).verdict if len(traj) >= MIN_TREND_N else "insufficient" + out[m.key] = ( + gate(traj).verdict if len(traj) >= MIN_TREND_N else Verdict.INSUFFICIENT + ) return out @@ -1745,7 +1783,7 @@ def compute_steady_state_metrics( anomaly=detect_level_shift(series, plateaus), short_window=None, global_trend=gt, - drifting_up=[k for k, v in gt.items() if v == "up"], + drifting_up=[k for k, v in gt.items() if v is Verdict.DRIFTING_UP], ) # Min-duration selection. When enforced, report the first plateau that # clears the gate. Earlier plateaus are too brief to certify. If none @@ -1846,7 +1884,7 @@ def compute_steady_state_metrics( anomaly=anomaly, short_window=short, global_trend=gt, - drifting_up=[k for k, v in gt.items() if v == "up"], + drifting_up=[k for k, v in gt.items() if v is Verdict.DRIFTING_UP], ) @@ -1913,10 +1951,10 @@ def run( # Rendering # --------------------------------------------------------------------------- # _VERDICT_GLYPH = { - "up": "^ up", - "down": "v down", - "steady": "= steady", - "insufficient": ". n/a", + Verdict.DRIFTING_UP: "^ up", + Verdict.DRIFTING_DOWN: "v down", + Verdict.PLATEAU: "= steady", + Verdict.INSUFFICIENT: ". n/a", } diff --git a/tests/unit/async_utils/services/metrics_aggregator/test_snapshot.py b/tests/unit/async_utils/services/metrics_aggregator/test_snapshot.py index 7fb304278..c612a85ef 100644 --- a/tests/unit/async_utils/services/metrics_aggregator/test_snapshot.py +++ b/tests/unit/async_utils/services/metrics_aggregator/test_snapshot.py @@ -20,6 +20,7 @@ import msgspec import msgspec.msgpack import pytest + from inference_endpoint.async_utils.services.metrics_aggregator.snapshot import ( METRICS_SNAPSHOT_TOPIC, CounterStat, @@ -35,6 +36,7 @@ SteadyState, SteadyWindow, TpsBlock, + Verdict, ) @@ -159,7 +161,7 @@ def test_steady_state_survives_the_wire_and_the_dict_form(self): plateaus=[], ), short_window=None, - global_trend={}, + global_trend={verdict.value: verdict for verdict in Verdict}, drifting_up=["ttft_p50"], ) snap = MetricsSnapshot( @@ -174,7 +176,12 @@ def test_steady_state_survives_the_wire_and_the_dict_form(self): _, payload = codec.encode(snap) # The wire keeps the Struct; the dict form is builtins, since that is # what final_snapshot.json holds and what Report.from_snapshot converts. - assert codec.decode(payload).steady_state == verdict + decoded = codec.decode(payload) + assert decoded.steady_state == verdict + assert all( + isinstance(value, Verdict) + for value in decoded.steady_state.global_trend.values() + ) assert snapshot_to_dict(snap)["steady_state"] == msgspec.to_builtins(verdict) def test_steady_state_defaults_to_none_for_older_frames(self): @@ -183,6 +190,37 @@ def test_steady_state_defaults_to_none_for_older_frames(self): payload = msgspec.msgpack.encode([4, 8, "live", 0, []]) assert codec.decode(payload).steady_state is None + @pytest.mark.parametrize("wire_verdict", ["up", "down", "steady", "unknown", 1]) + def test_invalid_trend_verdict_is_rejected_and_dropped(self, wire_verdict): + codec = MetricsSnapshotCodec() + verdict = SteadyState( + found=False, + reason="invalid trend", + superpass_size=1, + n_super_passes=4, + warmup=0, + window=None, + ttft=None, + tpot=None, + osl=None, + latency=None, + tps=None, + cov={}, + cov_basis=None, + anomaly=Anomaly(False, None, 0.0, None, []), + short_window=None, + global_trend={"tpot_p90": Verdict.DRIFTING_UP}, + drifting_up=["tpot_p90"], + ) + snap = MetricsSnapshot(1, 2, SessionState.COMPLETE, 0, [], verdict) + _, payload = codec.encode(snap) + raw = msgspec.msgpack.decode(payload) + raw[-1]["global_trend"]["tpot_p90"] = wire_verdict + invalid_payload = msgspec.msgpack.encode(raw) + with pytest.raises(msgspec.ValidationError) as error: + codec.decode(invalid_payload) + assert codec.on_decode_error(invalid_payload, error.value) is None + def test_non_finite_floats_inside_the_verdict_are_scrubbed(self): """Without this, json.dumps(allow_nan=False) costs the run its snapshot.""" snap = MetricsSnapshot( @@ -200,12 +238,10 @@ def test_non_finite_floats_inside_the_verdict_are_scrubbed(self): def test_on_decode_error_drops_malformed(self): codec = MetricsSnapshotCodec() - # Decode a clearly malformed payload (truncated msgpack) - try: - codec.decode(b"\xff\x00") - except Exception as e: - fallback = codec.on_decode_error(b"\xff\x00", e) - assert fallback is None + payload = b"\xff\x00" + with pytest.raises(msgspec.DecodeError) as error: + codec.decode(payload) + assert codec.on_decode_error(payload, error.value) is None def test_on_decode_error_reraises_unknown(self): codec = MetricsSnapshotCodec() diff --git a/tests/unit/metrics/test_report_builder.py b/tests/unit/metrics/test_report_builder.py index b284f983d..b0fd9d2b8 100644 --- a/tests/unit/metrics/test_report_builder.py +++ b/tests/unit/metrics/test_report_builder.py @@ -27,6 +27,7 @@ import msgspec.structs import pytest + from inference_endpoint.async_utils.services.metrics_aggregator.aggregator import ( MetricCounterKey, ) @@ -48,6 +49,7 @@ SteadyState, SteadyWindow, TpsBlock, + Verdict, ) # 1 hour in ns — same as the aggregator's default bound for time-series. @@ -1000,6 +1002,24 @@ def _report(self, verdict): def test_absent_by_default(self): assert self._report(None).steady_state is None + @pytest.mark.parametrize( + ("legacy", "canonical"), + [ + ("up", Verdict.DRIFTING_UP), + ("down", Verdict.DRIFTING_DOWN), + ("steady", Verdict.PLATEAU), + ], + ) + def test_legacy_global_trend_is_normalized_without_mutation( + self, legacy, canonical + ): + raw = msgspec.to_builtins(self._verdict()) + raw["global_trend"] = {"tpot_p90": legacy} + snap = {"steady_state": raw} + report = Report.from_snapshot(snap) + assert report.steady_state.global_trend["tpot_p90"] is canonical + assert raw["global_trend"]["tpot_p90"] == legacy + def test_read_from_the_snapshot_and_kept_in_to_json(self): verdict = self._verdict() report = self._report(verdict) diff --git a/tests/unit/metrics/test_steady_state_diagnostics.py b/tests/unit/metrics/test_steady_state_diagnostics.py index 5f151de55..90b8aa4a5 100644 --- a/tests/unit/metrics/test_steady_state_diagnostics.py +++ b/tests/unit/metrics/test_steady_state_diagnostics.py @@ -12,6 +12,7 @@ import json import pytest + from inference_endpoint.metrics import steady_state_diagnostics as mod pytestmark = pytest.mark.unit @@ -177,7 +178,7 @@ def test_cov_positive_for_varied_series(): # --------------------------------------------------------------------------- # -# Trend algorithms — verdict in {"up", "steady", "down"} +# Trend algorithms — verdict is a canonical enum. # --------------------------------------------------------------------------- # STRONG_UP = [float(i) for i in range(12)] @@ -186,15 +187,15 @@ def test_cov_positive_for_varied_series(): def test_mann_kendall_detects_upward_trend(): - assert mod.mann_kendall(STRONG_UP).verdict == "up" + assert mod.mann_kendall(STRONG_UP).verdict is mod.Verdict.DRIFTING_UP def test_mann_kendall_detects_downward_trend(): - assert mod.mann_kendall(STRONG_DOWN).verdict == "down" + assert mod.mann_kendall(STRONG_DOWN).verdict is mod.Verdict.DRIFTING_DOWN def test_mann_kendall_flat_is_steady(): - assert mod.mann_kendall(FLAT).verdict == "steady" + assert mod.mann_kendall(FLAT).verdict is mod.Verdict.PLATEAU def test_hamed_rao_inflates_variance_vs_plain_mk(): @@ -202,7 +203,7 @@ def test_hamed_rao_inflates_variance_vs_plain_mk(): plain = mod.mann_kendall(STRONG_UP) corrected = mod.mann_kendall_hamed_rao(STRONG_UP) assert corrected.variance >= plain.variance - assert corrected.verdict == "up" + assert corrected.verdict is mod.Verdict.DRIFTING_UP def test_hamed_rao_does_not_fabricate_trend_on_oscillation(): @@ -212,8 +213,8 @@ def test_hamed_rao_does_not_fabricate_trend_on_oscillation(): saw = [0.0, 20.0, 2.0, 22.0, 4.0, 24.0, 6.0, 26.0, 8.0, 28.0, 10.0, 30.0] plain = mod.mann_kendall(saw) hr = mod.mann_kendall_hamed_rao(saw) - assert plain.verdict == "up" # uncorrected MK false-positives - assert hr.verdict == "steady" # corrected test refuses + assert plain.verdict is mod.Verdict.DRIFTING_UP # uncorrected MK false-positives + assert hr.verdict is mod.Verdict.PLATEAU # corrected test refuses assert hr.variance >= plain.variance # inflated, not collapsed to a sliver @@ -222,25 +223,31 @@ def test_theil_sen_recovers_exact_slope(): def test_theil_sen_flat_is_steady(): - assert mod.theil_sen(FLAT).verdict == "steady" + assert mod.theil_sen(FLAT).verdict is mod.Verdict.PLATEAU def test_newey_west_detects_trend_and_flat(): - assert mod.newey_west(STRONG_UP).verdict == "up" - assert mod.newey_west(FLAT).verdict == "steady" + assert mod.newey_west(STRONG_UP).verdict is mod.Verdict.DRIFTING_UP + assert mod.newey_west(FLAT).verdict is mod.Verdict.PLATEAU def test_slope_vs_scatter_matches_reference_formula(): # Clean line: huge SNR, large rel-drift -> up. - assert mod.slope_vs_scatter(STRONG_UP).verdict == "up" - assert mod.slope_vs_scatter(STRONG_DOWN).verdict == "down" + assert mod.slope_vs_scatter(STRONG_UP).verdict is mod.Verdict.DRIFTING_UP + assert mod.slope_vs_scatter(STRONG_DOWN).verdict is mod.Verdict.DRIFTING_DOWN # Flat -> steady. - assert mod.slope_vs_scatter(FLAT).verdict == "steady" + assert mod.slope_vs_scatter(FLAT).verdict is mod.Verdict.PLATEAU def test_trend_algorithms_report_insufficient_below_min_n(): for fn in (mod.mann_kendall, mod.theil_sen, mod.newey_west, mod.slope_vs_scatter): - assert fn([1.0, 2.0, 3.0]).verdict == "insufficient" + assert fn([1.0, 2.0, 3.0]).verdict is mod.Verdict.INSUFFICIENT + + +def test_verdict_json_round_trip_is_canonical(): + encoded = mod.msgspec.json.encode(mod.Verdict.DRIFTING_UP) + assert encoded == b'"drifting_up"' + assert mod.msgspec.json.decode(encoded, type=mod.Verdict) is mod.Verdict.DRIFTING_UP # --------------------------------------------------------------------------- # @@ -291,7 +298,7 @@ def test_run_result_structure(tmp_path): result = mod.run(path, superpass_size=1, count_tokens=_words, warmup=1) assert result["steady_state"].n_super_passes == 8 assert result["steady_state"].warmup == 1 - assert result["drift"]["ttft_p50"]["mk_hamed_rao"] == "up" + assert result["drift"]["ttft_p50"]["mk_hamed_rao"] is mod.Verdict.DRIFTING_UP # CoV cells carry a pass/fail per bound and a gate flag. Admissibility gates on # TPOT only, so tpot_p50 is gated while ttft_p50 is now diagnostic. cov = result["steady_state"].cov From 65996db4d67adbef0125ecda9c3aa56d61d12fb7 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 01:32:49 -0700 Subject: [PATCH 03/18] feat(validation): load versioned cohort policies and plan typed checks --- src/inference_endpoint/validation/LICENSE.md | 176 +++++ src/inference_endpoint/validation/__init__.py | 24 + .../validation/catalogs/__init__.py | 3 + .../validation/catalogs/seed_sets.yaml | 18 + .../validation/catalogs/seeds.py | 110 ++++ src/inference_endpoint/validation/cohorts.py | 50 ++ .../validation/conditions.py | 117 ++++ src/inference_endpoint/validation/loader.py | 79 +++ src/inference_endpoint/validation/models.py | 98 +++ .../validation/operations.py | 89 +++ src/inference_endpoint/validation/planner.py | 193 ++++++ .../policies/2026-10-C1/catalog.yaml | 321 +++++++++ .../policies/2026-10-C1/curve_checks.yaml | 217 +++++++ .../policies/2026-10-C1/point_checks.yaml | 346 ++++++++++ .../2026-10-C1/submission_checks.yaml | 36 ++ .../policies/2026-10-C1/system_checks.yaml | 46 ++ .../validation/references.py | 56 ++ .../validation/schemas/__init__.py | 8 + .../validation/schemas/registry.py | 48 ++ .../validation/schemas/requirements_v1.py | 610 ++++++++++++++++++ .../validation/schemas/v1.py | 192 ++++++ .../validation/schemas/values_v1.py | 149 +++++ src/inference_endpoint/validation/types.py | 126 ++++ .../validation/vocabulary.py | 117 ++++ .../validation/policy-fingerprints.json | 97 +++ tests/unit/validation/test_cohorts.py | 110 ++++ tests/unit/validation/test_coverage.py | 79 +++ tests/unit/validation/test_loader.py | 248 +++++++ tests/unit/validation/test_planner.py | 207 ++++++ .../unit/validation/test_policy_contracts.py | 98 +++ tests/unit/validation/test_seed_catalog.py | 246 +++++++ 31 files changed, 4314 insertions(+) create mode 100644 src/inference_endpoint/validation/LICENSE.md create mode 100644 src/inference_endpoint/validation/__init__.py create mode 100644 src/inference_endpoint/validation/catalogs/__init__.py create mode 100644 src/inference_endpoint/validation/catalogs/seed_sets.yaml create mode 100644 src/inference_endpoint/validation/catalogs/seeds.py create mode 100644 src/inference_endpoint/validation/cohorts.py create mode 100644 src/inference_endpoint/validation/conditions.py create mode 100644 src/inference_endpoint/validation/loader.py create mode 100644 src/inference_endpoint/validation/models.py create mode 100644 src/inference_endpoint/validation/operations.py create mode 100644 src/inference_endpoint/validation/planner.py create mode 100644 src/inference_endpoint/validation/policies/2026-10-C1/catalog.yaml create mode 100644 src/inference_endpoint/validation/policies/2026-10-C1/curve_checks.yaml create mode 100644 src/inference_endpoint/validation/policies/2026-10-C1/point_checks.yaml create mode 100644 src/inference_endpoint/validation/policies/2026-10-C1/submission_checks.yaml create mode 100644 src/inference_endpoint/validation/policies/2026-10-C1/system_checks.yaml create mode 100644 src/inference_endpoint/validation/references.py create mode 100644 src/inference_endpoint/validation/schemas/__init__.py create mode 100644 src/inference_endpoint/validation/schemas/registry.py create mode 100644 src/inference_endpoint/validation/schemas/requirements_v1.py create mode 100644 src/inference_endpoint/validation/schemas/v1.py create mode 100644 src/inference_endpoint/validation/schemas/values_v1.py create mode 100644 src/inference_endpoint/validation/types.py create mode 100644 src/inference_endpoint/validation/vocabulary.py create mode 100644 tests/fixtures/validation/policy-fingerprints.json create mode 100644 tests/unit/validation/test_cohorts.py create mode 100644 tests/unit/validation/test_coverage.py create mode 100644 tests/unit/validation/test_loader.py create mode 100644 tests/unit/validation/test_planner.py create mode 100644 tests/unit/validation/test_policy_contracts.py create mode 100644 tests/unit/validation/test_seed_catalog.py diff --git a/src/inference_endpoint/validation/LICENSE.md b/src/inference_endpoint/validation/LICENSE.md new file mode 100644 index 000000000..8318dc07b --- /dev/null +++ b/src/inference_endpoint/validation/LICENSE.md @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/src/inference_endpoint/validation/__init__.py b/src/inference_endpoint/validation/__init__.py new file mode 100644 index 000000000..922321be9 --- /dev/null +++ b/src/inference_endpoint/validation/__init__.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Versioned submission-validation policies and typed check planning.""" + +from .conditions import Conditions, Context +from .loader import bundled_policy_path, load_policy +from .models import Policy, Rule +from .planner import CheckPlan, PlannedCheck, plan_checks +from .types import Decision, EvidenceKey, Scope + +__all__ = [ + "CheckPlan", + "Conditions", + "Context", + "Decision", + "EvidenceKey", + "PlannedCheck", + "Policy", + "Rule", + "Scope", + "bundled_policy_path", + "load_policy", + "plan_checks", +] diff --git a/src/inference_endpoint/validation/catalogs/__init__.py b/src/inference_endpoint/validation/catalogs/__init__.py new file mode 100644 index 000000000..521e1858d --- /dev/null +++ b/src/inference_endpoint/validation/catalogs/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Published validation catalogs.""" diff --git a/src/inference_endpoint/validation/catalogs/seed_sets.yaml b/src/inference_endpoint/validation/catalogs/seed_sets.yaml new file mode 100644 index 000000000..e3dcaf2ac --- /dev/null +++ b/src/inference_endpoint/validation/catalogs/seed_sets.yaml @@ -0,0 +1,18 @@ +# MLPerf Endpoints published seed sets (§4.6 Seed Rotation). +# +# Mirrored verbatim from `mlcommons/endpoints_policies` `seedset.yaml`, merged to +# `v1.0_rules_dev` in PR #117 (`a36d158`, 2026-09-15). +# +# `cohort-id` is the cohort the sets below were published for. §4.6 makes each set +# adoptable for that cohort and the three following it, so the adoption window is +# derived rather than listed. +# + +cohort: + version: 1.0 + cohort-id: 2026-10-C1 + seed_sets: # a submission selects exactly one, by id + - id: A + scheduler_rng_seed: 10487924139932647040 + sample_index_rng_seed: 586478644936801402 + model_seed: 9315206023656308754 diff --git a/src/inference_endpoint/validation/catalogs/seeds.py b/src/inference_endpoint/validation/catalogs/seeds.py new file mode 100644 index 000000000..4e482ec13 --- /dev/null +++ b/src/inference_endpoint/validation/catalogs/seeds.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Load published seed values and cohort adoption windows.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +from ..cohorts import Cohort, adoption_window + +_BUNDLED = Path(__file__).parent / "seed_sets.yaml" +_SEED_FIELDS = ("scheduler_rng_seed", "sample_index_rng_seed", "model_seed") + + +class SeedSetError(ValueError): + """Raised when a seed-set file cannot be read or is malformed.""" + + +@dataclass(frozen=True) +class SeedSet: + """One published seed set.""" + + id: str + scheduler_rng_seed: int + sample_index_rng_seed: int + model_seed: int + cohorts: tuple[str, ...] = field(default_factory=tuple) + + @property + def seeds(self) -> dict[str, int]: + """The set's seeds keyed by field name, for comparison against a point.""" + return {name: getattr(self, name) for name in _SEED_FIELDS} + + +def bundled_seed_sets_path() -> Path: + """Path to the seed-set file shipped with this checker.""" + return _BUNDLED + + +def load_seed_sets() -> dict[str, SeedSet]: + """Load the bundled published seed catalog.""" + return parse_seed_catalog(_BUNDLED) + + +def parse_seed_catalog(chosen: Path) -> dict[str, SeedSet]: + """Parse a catalog and derive its publication adoption windows.""" + try: + raw = yaml.safe_load(chosen.read_text(encoding="utf-8")) + except (OSError, UnicodeError) as exc: + raise SeedSetError(f"Cannot read seed-set file {chosen}: {exc}") from exc + except yaml.YAMLError as exc: + raise SeedSetError(f"Invalid YAML in seed-set file {chosen}: {exc}") from exc + entries, published = _unwrap(chosen, raw) + cohorts: tuple[str, ...] = () + if published is not None: + parsed = Cohort.parse(published) + if parsed is None: + raise SeedSetError( + f"{chosen}: cohort-id {published!r} is not of the form YYYY-MM-C0/C1" + ) + cohorts = tuple(str(c) for c in adoption_window(parsed)) + sets: dict[str, SeedSet] = {} + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + raise SeedSetError(f"{chosen}: seed_sets[{index}] is not a mapping") + set_id = entry.get("id") + if not isinstance(set_id, str) or not set_id: + raise SeedSetError(f"{chosen}: seed_sets[{index}] has no 'id'") + if set_id in sets: + raise SeedSetError(f"{chosen}: duplicate seed set {set_id!r}") + try: + seeds = {name: entry[name] for name in _SEED_FIELDS} + if any(type(value) is not int for value in seeds.values()): + raise ValueError("Seeds must be integers without coercion") + except (KeyError, TypeError, ValueError) as exc: + raise SeedSetError( + f"{chosen}: seed set {set_id!r} must define integer {', '.join(_SEED_FIELDS)}: {exc}" + ) from exc + explicit = entry.get("cohorts") + if explicit is not None and (not isinstance(explicit, list)): + raise SeedSetError( + f"{chosen}: seed set {set_id!r} has a non-list 'cohorts'" + ) + window = tuple(str(c) for c in explicit) if explicit is not None else cohorts + sets[set_id] = SeedSet(id=set_id, cohorts=window, **seeds) + if not sets: + raise SeedSetError(f"{chosen} defines no seed sets") + return sets + + +def _unwrap(path: Path, raw: object) -> tuple[list[object], str | None]: + """Return ``(seed-set entries, publication cohort)`` from either registry shape.""" + if not isinstance(raw, dict): + raise SeedSetError(f"{path} must be a mapping") + cohort = raw.get("cohort") + if isinstance(cohort, dict): + entries = cohort.get("seed_sets") + published = cohort.get("cohort-id") + if not isinstance(entries, list): + raise SeedSetError(f"{path}: cohort.seed_sets must be a list") + return (entries, str(published) if published is not None else None) + entries = raw.get("seed_sets") + if not isinstance(entries, list): + raise SeedSetError( + f"{path} must define cohort.seed_sets (or a top-level seed_sets list)" + ) + return (entries, None) diff --git a/src/inference_endpoint/validation/cohorts.py b/src/inference_endpoint/validation/cohorts.py new file mode 100644 index 000000000..f4c78588c --- /dev/null +++ b/src/inference_endpoint/validation/cohorts.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Publication cohort identifiers and adoption windows.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +COHORT_RE = re.compile("(?P\\d{4})-(?P\\d{2})-C(?P[01])") +_COHORTS_PER_MONTH = 2 + + +@dataclass(frozen=True, order=True) +class Cohort: + """One publication cohort.""" + + year: int + month: int + index: int + + def __str__(self) -> str: + return f"{self.year:04d}-{self.month:02d}-C{self.index}" + + @classmethod + def parse(cls, value: str) -> Cohort | None: + """Parse ``YYYY-MM-C0`` / ``YYYY-MM-C1``, returning None when malformed.""" + match = COHORT_RE.fullmatch(value.strip()) if value else None + if match is None: + return None + month = int(match["month"]) + if not 1 <= month <= 12: + return None + return cls(year=int(match["year"]), month=month, index=int(match["index"])) + + def next(self) -> Cohort: + """The cohort immediately following this one.""" + if self.index + 1 < _COHORTS_PER_MONTH: + return Cohort(self.year, self.month, self.index + 1) + if self.month == 12: + return Cohort(self.year + 1, 1, 0) + return Cohort(self.year, self.month + 1, 0) + + +def adoption_window(published: Cohort, length: int = 4) -> tuple[Cohort, ...]: + """The cohorts during which a set published in *published* may be adopted.""" + window = [published] + while len(window) < length: + window.append(window[-1].next()) + return tuple(window) diff --git a/src/inference_endpoint/validation/conditions.py b/src/inference_endpoint/validation/conditions.py new file mode 100644 index 000000000..ee64c7b26 --- /dev/null +++ b/src/inference_endpoint/validation/conditions.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Typed inclusion and exclusion predicates over classified subjects.""" + +from pydantic import Field, model_validator + +from inference_endpoint.config.schema import LoadPatternType + +from .types import ( + Declaration, + Division, + EvidenceKey, + FrozenModel, + Identifier, + Match, + OfflineMode, + Scope, +) + + +class Context(FrozenModel): + """A subject classified by the artifact adapter; None denotes unknown facts. + + Known absence is represented by empty sets or False. Available dependencies + must have passed artifact parsing/computation; presence alone is insufficient. + """ + + id: Identifier + scope: Scope + model_id: Identifier | None = None + load_pattern: LoadPatternType | None = None + offline: OfflineMode | None = None + division: Division | None = None + declared: frozenset[Declaration] | None = None + speculative_decoding: bool | None = Field(default=None, strict=True) + available: frozenset[EvidenceKey] = frozenset() + invalid: frozenset[EvidenceKey] | None = None + members: tuple["Context", ...] | None = None + + @model_validator(mode="after") + def validate_evidence(self): + if self.invalid and self.invalid & self.available: + raise ValueError("Invalid evidence cannot also be available") + if self.members is not None: + if self.scope is not Scope.CURVE or any( + member.scope is not Scope.POINT for member in self.members + ): + raise ValueError("Curve members must be point contexts") + if len({member.id for member in self.members}) != len(self.members): + raise ValueError("Duplicate curve-member IDs") + return self + + +class Conditions(FrozenModel): + """Different fields are ANDed; enum lists are alternatives except fact sets.""" + + models: tuple[Identifier, ...] | None = Field(default=None, min_length=1) + load_pattern: tuple[LoadPatternType, ...] | None = Field(default=None, min_length=1) + offline: tuple[OfflineMode, ...] | None = Field(default=None, min_length=1) + division: tuple[Division, ...] | None = Field(default=None, min_length=1) + declared: frozenset[Declaration] | None = Field(default=None, min_length=1) + not_declared: frozenset[Declaration] | None = Field(default=None, min_length=1) + invalid: frozenset[EvidenceKey] | None = Field(default=None, min_length=1) + speculative_decoding: bool | None = Field(default=None, strict=True) + + @model_validator(mode="after") + def validate_predicates(self): + if not self.model_fields_set or any( + getattr(self, field) is None for field in self.model_fields_set + ): + raise ValueError("Conditions require non-null predicates") + if self.declared and self.not_declared and self.declared & self.not_declared: + raise ValueError("A declaration cannot be both required and excluded") + return self + + def __call__(self, context: Context) -> Match: + results: list[Match] = [] + for required, actual in ( + (self.models, context.model_id), + (self.load_pattern, context.load_pattern), + (self.offline, context.offline), + (self.division, context.division), + ): + if required is not None: + results.append( + Match.UNKNOWN + if actual is None + else Match.YES + if actual in required + else Match.NO + ) + for required_set, actual_set, exclude in ( + (self.declared, context.declared, False), + (self.not_declared, context.declared, True), + (self.invalid, context.invalid, False), + ): + if required_set is not None: + if actual_set is None: + results.append(Match.UNKNOWN) + else: + matches = ( + not bool(required_set.intersection(actual_set)) + if exclude + else required_set <= actual_set + ) + results.append(Match.YES if matches else Match.NO) + if self.speculative_decoding is not None: + results.append( + Match.UNKNOWN + if context.speculative_decoding is None + else Match.YES + if context.speculative_decoding == self.speculative_decoding + else Match.NO + ) + if Match.NO in results: + return Match.NO + return Match.UNKNOWN if Match.UNKNOWN in results else Match.YES diff --git a/src/inference_endpoint/validation/loader.py b/src/inference_endpoint/validation/loader.py new file mode 100644 index 000000000..8ccc03f85 --- /dev/null +++ b/src/inference_endpoint/validation/loader.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Load an atomic five-file policy bundle, retaining hashes for reproducibility.""" + +import hashlib +import json +from os import PathLike +from pathlib import Path + +import yaml + +from .models import Policy +from .schemas import parser_for +from .types import PolicyFile, Release + + +class UniqueKeyLoader(yaml.SafeLoader): + def construct_mapping(self, node, deep=False): + self.flatten_mapping(node) + result = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + if not isinstance(key, str): + raise ValueError("Policy mapping keys must be strings") + if key in result: + raise ValueError(f"Duplicate YAML key {key}") + result[key] = self.construct_object(value_node, deep=deep) + return result + + +def bundled_policy_path(version: str = "2026-10-C1") -> Path: + Release(version=version, revision=1) + return Path(__file__).parent / "policies" / version + + +def load_policy(directory: PathLike[str]) -> Policy: + directory = Path(directory) + if not directory.is_dir(): + raise ValueError(f"Policy directory does not exist: {directory}") + expected = {file.value for file in PolicyFile} + actual = { + path.name for path in directory.iterdir() if path.suffix in {".yaml", ".yml"} + } + if actual != expected: + raise ValueError( + f"Invalid policy files; missing={sorted(expected - actual)}, unexpected={sorted(actual - expected)}" + ) + documents = {} + digests = {} + release = None + for file in PolicyFile: + path = directory / file.value + try: + raw = path.read_bytes() + document = yaml.load(raw, Loader=UniqueKeyLoader) + if not isinstance(document, dict): + raise ValueError("Expected a policy mapping") + json.dumps(document, allow_nan=False) + identity = Release.model_validate( + { + key: document[key] + for key in ("version", "revision") + if key in document + } + ) + if release is not None and identity != release: + raise ValueError("All policy files must have the same version/revision") + release = identity + documents[file] = document + digests[file] = hashlib.sha256(raw).hexdigest() + except (OSError, ValueError, TypeError, yaml.YAMLError) as error: + raise ValueError(f"Invalid {file.value}: {error}") from error + assert release is not None + if directory.name != release.version: + raise ValueError("Policy directory name must match its cohort version") + try: + return parser_for(release)(documents, digests) + except ValueError as error: + raise ValueError(f"Invalid policy bundle: {error}") from error diff --git a/src/inference_endpoint/validation/models.py b/src/inference_endpoint/validation/models.py new file mode 100644 index 000000000..c601a5218 --- /dev/null +++ b/src/inference_endpoint/validation/models.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Immutable policy definitions independent of file layout and parser versions.""" + +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Any, TypeVar, overload + +from .conditions import Conditions +from .types import EvidenceKey, PolicyFile, Release, Scope, Severity +from .vocabulary import CheckKind, EvidenceReference + +_Key = TypeVar("_Key") + + +@overload +def freeze( # noqa: UP047 - supported by the repository type checker + value: Mapping[_Key, Any], +) -> Mapping[_Key, Any]: ... + + +@overload +def freeze(value: object) -> Any: ... + + +def freeze(value: object) -> Any: + if isinstance(value, dict): + return MappingProxyType({key: freeze(child) for key, child in value.items()}) + if isinstance(value, list): + return tuple(freeze(child) for child in value) + return value + + +def thaw(value: object) -> Any: + if isinstance(value, Mapping): + return {key: thaw(child) for key, child in value.items()} + if isinstance(value, tuple): + return [thaw(child) for child in value] + return value + + +@dataclass(frozen=True) +class Rule: + id: str + kind: CheckKind + scope: Scope + severity: Severity + applies_to: Conditions | None + unless: Conditions | None + requires: frozenset[EvidenceKey] + requirements: Mapping[str, Any] + references: frozenset[EvidenceReference] + source: PolicyFile + needs_model: bool = False + + +@dataclass(frozen=True) +class Override: + id: str + rule: str + reason: str + applies_to: Conditions + enabled: bool + requirements: Mapping[str, Any] + + +@dataclass(frozen=True) +class Policy: + release: Release + catalogs: Mapping[str, Any] + models: frozenset[str] + checks: tuple[Rule, ...] + overrides: tuple[Override, ...] + documents: Mapping[PolicyFile, Mapping[str, Any]] + source_digests: Mapping[PolicyFile, str] + digest: str + + def as_monolith(self) -> dict: + catalog = thaw(self.documents[PolicyFile.CATALOG]) + groups = {} + for name in PolicyFile: + if name is PolicyFile.CATALOG: + continue + document = thaw(self.documents[name]) + groups[name.scope.value] = { + key: value + for key, value in document.items() + if key not in {"version", "revision"} + } + groups["diagnostics"] = catalog["diagnostics"] + return { + "version": self.release.version, + "revision": self.release.revision, + "catalogs": catalog["catalogs"], + "check_groups": groups, + "enrollment": catalog["enrollment"], + } diff --git a/src/inference_endpoint/validation/operations.py b/src/inference_endpoint/validation/operations.py new file mode 100644 index 000000000..14f101bce --- /dev/null +++ b/src/inference_endpoint/validation/operations.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Operation vocabulary shared by policy contracts and callable evaluators.""" + +from enum import StrEnum + + +class BindingMatch(StrEnum): + CHECKPOINT = "model_and_repository_and_revision" + CLIENT_ALLOWLIST = "exact_revision" + + +class AccuracyOperation(StrEnum): + SINGLE_TURN_METRICS = "single_turn_metrics" + ISSUED_COUNT = "issued_count" + AGENTIC_PROFILE_AVAILABLE = "agentic_profile_available" + PER_POINT_FRACTION = "per_point_fraction" + MEAN_OF_BAND_MEANS = "mean_of_band_means" + PER_POINT_RANGE = "per_point_range" + + +class RegionPlacementOperation(StrEnum): + IN_RANGE = "in_range" + DECLARED_MATCHES_COMPUTED = "declared_matches_computed" + + +class OfflineOperation(StrEnum): + DECLARATION_COUNT = "declaration_count" + ORDERING = "ordering" + + +class DrafterOperation(StrEnum): + MEMBERSHIP = "membership" + APPROVAL_AGE = "approval_age" + + +class ComparisonOperator(StrEnum): + GREATER_THAN = "greater_than" + GREATER_THAN_OR_EQUAL = "greater_than_or_equal" + EQUAL = "equal" + + +class OperandKind(StrEnum): + FIELD = "field" + CONSTANT = "constant" + + +class DerivedOperation(StrEnum): + CURVE_PEAK_RATIO = "curve_peak_ratio" + OUTPUT_TOKENS_PER_ELAPSED_SECOND = "output_tokens_per_elapsed_second" + INVERSE_TPOT_P90_MS = "inverse_tpot_p90_ms" + THROUGHPUT_PER_PROVISIONED_KW = "throughput_per_provisioned_kw" + OUTPUT_TOKENS_PER_COMPLETED_TURN_SECOND = "output_tokens_per_completed_turn_second" + + +class PowerOperation(StrEnum): + DESCRIPTOR = "descriptor" + APPENDIX_D_DEFAULTS = "appendix_d_defaults" + ENGAGED_NODES = "engaged_nodes" + NODE_DECLARATION = "node_declaration" + MAXIMAL_ENGAGEMENT = "maximal_engagement" + + +class SeedOperation(StrEnum): + LEGACY_NAMES = "legacy_names" + MEMBERSHIP = "membership" + RUNTIME_VALUES = "runtime_values" + ADOPTION_WINDOW = "adoption_window" + + +class SteadyOperation(StrEnum): + VOCABULARY = "vocabulary" + STATUS_WINDOW_AGREEMENT = "status_window_agreement" + REPORTING_BASIS = "reporting_basis" + + +class WarmupOperation(StrEnum): + LOG_RETENTION = "log_retention" + SALT_DISABLED = "salt_disabled" + + +class DurationBasis(StrEnum): + REPORTED_WINDOW_ISSUE_SPAN = "reported_window_issue_span" + WHOLE_RUN_DURATION = "whole_run_duration" + + +class DatasetCountOperator(StrEnum): + EQUAL = "equal" + POSITIVE_MULTIPLE = "positive_multiple" diff --git a/src/inference_endpoint/validation/planner.py b/src/inference_endpoint/validation/planner.py new file mode 100644 index 000000000..11342840e --- /dev/null +++ b/src/inference_endpoint/validation/planner.py @@ -0,0 +1,193 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Select scoped checks and resolve prerequisites without evaluating compliance.""" + +from collections.abc import Iterable +from dataclasses import dataclass, replace +from types import MappingProxyType + +from .conditions import Context +from .models import Override, Policy, Rule +from .references import evidence_references, needs_model +from .types import Decision, EvidenceKey, Match + + +@dataclass(frozen=True) +class PlannedCheck: + subject: Context + rule: Rule + decision: Decision + reason: str + missing: frozenset[EvidenceKey] = frozenset() + override: str | None = None + selected_members: tuple[str, ...] | None = None + + +@dataclass(frozen=True) +class CheckPlan: + policy_digest: str + checks: tuple[PlannedCheck, ...] + + @property + def ready(self) -> tuple[PlannedCheck, ...]: + return tuple(check for check in self.checks if check.decision is Decision.READY) + + @property + def blocked(self) -> tuple[PlannedCheck, ...]: + return tuple( + check for check in self.checks if check.decision is Decision.BLOCKED + ) + + +def _select(rule: Rule, subject: Context) -> tuple[Decision, str]: + included = rule.applies_to(subject) if rule.applies_to else Match.YES + excluded = rule.unless(subject) if rule.unless else Match.NO + if excluded is Match.YES: + return Decision.EXCLUDED, "Exclusion conditions match" + if included is Match.NO: + return Decision.EXCLUDED, "Inclusion conditions do not match" + if Match.UNKNOWN in (included, excluded): + return ( + Decision.BLOCKED, + "Classification facts required for selection are unknown", + ) + return Decision.READY, "Applicable" + + +def _member_subject(subject: Context, member: Context) -> Context: + """Inherit shared identity while preserving unknown point-specific facts.""" + fields = ("model_id", "division") + return member.model_copy( + update={ + field: getattr(subject, field) + for field in fields + if getattr(member, field) is None + } + ) + + +def _select_collection(rule: Rule, subject: Context): + if subject.members is None or not (rule.applies_to or rule.unless): + return (*_select(rule, subject), None) + selected = [] + unknown = False + for member in subject.members: + decision, _reason = _select(rule, _member_subject(subject, member)) + if decision is Decision.READY: + selected.append(member.id) + elif decision is Decision.BLOCKED: + unknown = True + if unknown: + return ( + Decision.BLOCKED, + "Collection selection needs member classification facts", + tuple(selected), + ) + if not selected: + return Decision.EXCLUDED, "No collection members match", () + return Decision.READY, "Applicable to selected collection members", tuple(selected) + + +def _override_partitions( + policy: Policy, + rule: Rule, + subject: Context, + selected_members: tuple[str, ...] | None, +) -> list[tuple[Override | None, bool, tuple[str, ...] | None]]: + overrides = [override for override in policy.overrides if override.rule == rule.id] + if not overrides: + return [(None, False, selected_members)] + member_selection = bool(subject.members) + contexts = ( + [ + _member_subject(subject, member) + for member in (subject.members or ()) + if selected_members is None or member.id in selected_members + ] + if member_selection + else [subject] + ) + partitions: dict[tuple[str | None, bool], tuple[Override | None, list[str]]] = {} + for context in contexts: + matches = [] + unknown = False + for override in overrides: + match = override.applies_to(context) + if match is Match.YES: + matches.append(override) + elif match is Match.UNKNOWN: + unknown = True + if len(matches) > 1: + raise ValueError(f"Overlapping overrides for {rule.id} on {context.id}") + selected_override = matches[0] if matches and not unknown else None + key = (selected_override.id if selected_override else None, unknown) + if key not in partitions: + partitions[key] = (selected_override, []) + partitions[key][1].append(context.id) + return [ + (override, unknown, tuple(members) if member_selection else selected_members) + for (_, unknown), (override, members) in partitions.items() + ] + + +def plan_checks(policy: Policy, subjects: Iterable[Context]) -> CheckPlan: + subjects = tuple(subjects) + if not subjects: + raise ValueError("Planning requires at least one classified subject") + if len({(subject.scope, subject.id) for subject in subjects}) != len(subjects): + raise ValueError("Duplicate scoped subject IDs") + planned = [] + for subject in subjects: + for original in policy.checks: + if original.scope is not subject.scope: + continue + selection, selection_reason, selected_members = _select_collection( + original, subject + ) + partitions = ( + _override_partitions(policy, original, subject, selected_members) + if selection is Decision.READY + else [(None, False, selected_members)] + ) + for override, unresolved, members in partitions: + rule = original + decision, reason = selection, selection_reason + missing: frozenset[EvidenceKey] = frozenset() + applied = None + if unresolved: + decision, reason = ( + Decision.BLOCKED, + "Override selection needs classification facts", + ) + elif override: + applied = override.id + if not override.enabled: + decision, reason = Decision.EXCLUDED, override.reason + else: + requirements = {**rule.requirements, **override.requirements} + rule = replace( + rule, + requirements=MappingProxyType(requirements), + references=evidence_references(requirements), + needs_model=needs_model(requirements), + ) + reason = override.reason + if decision is Decision.READY: + if rule.needs_model and subject.model_id not in policy.models: + decision, reason = ( + Decision.BLOCKED, + "Model is missing or not enrolled", + ) + else: + missing = rule.requires - subject.available + if missing: + decision, reason = ( + Decision.BLOCKED, + "Required evidence is missing or invalid", + ) + planned.append( + PlannedCheck( + subject, rule, decision, reason, missing, applied, members + ) + ) + return CheckPlan(policy.digest, tuple(planned)) diff --git a/src/inference_endpoint/validation/policies/2026-10-C1/catalog.yaml b/src/inference_endpoint/validation/policies/2026-10-C1/catalog.yaml new file mode 100644 index 000000000..c148ec8cc --- /dev/null +++ b/src/inference_endpoint/validation/policies/2026-10-C1/catalog.yaml @@ -0,0 +1,321 @@ +# Cohort policy; all five files must pin the same version and revision. +version: 2026-10-C1 +revision: 1 +catalogs: + load_pattern: [concurrency, agentic_inference, max_throughput] + divisions: [Standardized, Serviced, RDI] + offline_modes: [none, dedicated, elected] + region_names: + [low_latency, low_concurrency, med_concurrency, high_concurrency, margin] + declared_regions: + [ + low_latency, + low_concurrency, + med_concurrency, + high_concurrency, + margin, + submitters_choice, + ] + mandatory_accuracy_bands: + [ultra_low_concurrency, low_concurrency, med_concurrency, high_concurrency] + datasets: + open_orca: { sample_count: 24576, is_legacy: true } + cnn_dailymail: { sample_count: 13368 } + aime25: { sample_count: 30 } + gpqa: { sample_count: 198 } + livecodebench: { sample_count: 880 } + shopify_product_catalogue: { sample_count: 48289 } + shopify_product_catalogue_8k: { sample_count: 8000 } + mlperf_gpt_oss_performance: { sample_count: 6396 } + mlperf_gpt_oss_accuracy: { sample_count: 4395 } + math500: { sample_count: 500 } + legacy_mlperf_deepseek_r1: { sample_count: 4388, is_legacy: true } + legacy_llama3_1_405b_accuracy: { sample_count: 8313, is_legacy: true } + legacy_mixtral_accuracy: { sample_count: 15000, is_legacy: true } + agentic_combined: { sample_count: 613, sample_unit: trajectory } + swe_bench: { sample_count: 200, sample_unit: instance } + models: + llama3_1-8b: + accuracy: + kind: single_turn + datasets: [cnn_dailymail] + metrics: + rouge1: { reference: 38.7792, minimum_multiplier: 0.99 } + rouge2: { reference: 15.9075, minimum_multiplier: 0.99 } + rougel: { reference: 24.4957, minimum_multiplier: 0.99 } + rougelsum: { reference: 35.793, minimum_multiplier: 0.99 } + gen_len: + { + reference: 8167644, + minimum_multiplier: 0.9, + maximum_multiplier: 1.1, + } + gpt-oss-120b: + accuracy: + { + kind: single_turn, + datasets: [mlperf_gpt_oss_accuracy], + metrics: + { exact_match: { reference: 83.13, minimum_multiplier: 0.99 } }, + } + deepseek-r1: + accuracy: + { + kind: single_turn, + datasets: [legacy_mlperf_deepseek_r1], + metrics: + { exact_match: { reference: 81.3582, minimum_multiplier: 0.99 } }, + } + kimi-k3: + accuracy: + kind: agentic + datasets: [agentic_combined, swe_bench] + inline_minimum_percent: 58.32 + swebench_mean_minimum_percent: 93.5 + full_run_osl_range: [425, 520] + qwen3_6-35b-a3b: + accuracy: + kind: agentic + datasets: [agentic_combined, swe_bench] + inline_minimum_percent: 55.86 + swebench_mean_minimum_percent: 69.0 + full_run_osl_range: [344, 422] + deepseek-v4_1-flash: + accuracy: + kind: agentic + datasets: [agentic_combined, swe_bench] + inline_minimum_percent: 52.36 + swebench_mean_minimum_percent: 96.4 + full_run_osl_range: [793, 970] + llama3_1-405b: + accuracy: + kind: single_turn + datasets: [legacy_llama3_1_405b_accuracy] + metrics: + rougel: { reference: 21.6666, minimum_multiplier: 0.99 } + exact_match: { reference: 90.1335, minimum_multiplier: 0.99 } + tokens_per_sample: + { + reference: 684.68, + minimum_multiplier: 0.9, + maximum_multiplier: 1.1, + } + llama2-70b: + accuracy: + kind: single_turn + datasets: [open_orca] + metrics: + rouge1: { reference: 44.4312, minimum_multiplier: 0.99 } + rouge2: { reference: 22.0352, minimum_multiplier: 0.99 } + rougel: { reference: 28.6162, minimum_multiplier: 0.99 } + tokens_per_sample: + { + reference: 294.45, + minimum_multiplier: 0.9, + maximum_multiplier: 1.1, + } + mixtral-8x7b: + accuracy: + kind: single_turn + datasets: [legacy_mixtral_accuracy] + metrics: + rouge1: { reference: 45.5989, minimum_multiplier: 0.99 } + rouge2: { reference: 23.3526, minimum_multiplier: 0.99 } + rougel: { reference: 30.4608, minimum_multiplier: 0.99 } + tokens_per_sample: + { + minimum_reference: 144.84, + maximum_reference: 145.9, + minimum_multiplier: 0.9, + maximum_multiplier: 1.1, + } + approved_sped_decode_heads: + - model: kimi-k3 + repository: RadixArk/Kimi-K3-DSpark + revision: 3c5bac301d9cf392706189d82ed947feca6c2f0f + method: dspark + - model: kimi-k3 + repository: Inferact/Kimi-K3-DSpark + revision: cf6b8244620e7ea4b0651d214f28e89eac75bed6 + method: dspark + - model: deepseek-v4_1-flash + repository: deepseek-ai/DeepSeek-V4.1-Flash + revision: dba1be0a40aa45a94ad051997016db3960a90277 + method: dspark + bundled: true + - model: qwen3_6-35b-a3b + repository: Qwen/Qwen3.6-35B-A3B + revision: 995ad96eacd98c81ed38be0c5b274b04031597b0 + method: mtp + bundled: true + - model: qwen3_6-35b-a3b + repository: Qwen/Qwen3.6-35B-A3B-FP8 + revision: 95a723d08a9490559dae23d0cff1d9466213d989 + method: mtp + bundled: true + - model: qwen3_6-35b-a3b + repository: nvidia/Qwen3.6-35B-A3B-NVFP4 + revision: 1355db6a052410cfd62085d94b58866fd0f2c3c5 + method: mtp + bundled: true + approved_client_revisions: [] + approved_checkpoints: + llama3_1-8b: + - repository: meta-llama/Llama-3.1-8B-Instruct + revision: 0e9e39f249a16976918f6564b8830bc894c89659 + gpt-oss-120b: + - repository: openai/gpt-oss-120b + revision: b5c939de8f754692c1647ca79fbf85e8c1e70f8a + - repository: openai/gpt-oss-120b + revision: 8b193b0ef83bd41b40eb71fee8f1432315e02a3e + deepseek-r1: + - repository: deepseek-ai/DeepSeek-R1 + revision: 56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad + - repository: deepseek-ai/DeepSeek-R1 + revision: a157fa3d494497a54586a333a23df6c2143e7697 + kimi-k3: + - repository: moonshotai/Kimi-K3 + revision: f831ab66814297da540d832a5235f8e904f29d06 + - repository: moonshotai/Kimi-K3 + revision: a590ce090cb049c93a33dfe8c208ec652aa20503 + - repository: nvidia/Kimi-K3-NVFP4 + revision: b2428a0b83a8b712ff2e1a8448a103d4175341f1 + - repository: nvidia/Kimi-K3-NVFP4 + revision: f8c5234a0a880bcc6cbf779a315e7ee2f405b812 + deepseek-v4_1-flash: + - repository: deepseek-ai/DeepSeek-V4.1-Flash + revision: 2cba9e42aa026125f3ed06c6d98c1db82f7ca027 + - repository: deepseek-ai/DeepSeek-V4.1-Flash + revision: dba1be0a40aa45a94ad051997016db3960a90277 + qwen3_6-35b-a3b: + - repository: Qwen/Qwen3.6-35B-A3B + revision: 995ad96eacd98c81ed38be0c5b274b04031597b0 + - repository: Qwen/Qwen3.6-35B-A3B + revision: 53c43178507d69762986fbfa314f6e8d4d859409 + - repository: Qwen/Qwen3.6-35B-A3B-FP8 + revision: 95a723d08a9490559dae23d0cff1d9466213d989 + - repository: Qwen/Qwen3.6-35B-A3B-FP8 + revision: b2c1da757f2269109e990d56aeb3f2460260db3b + - repository: nvidia/Qwen3.6-35B-A3B-NVFP4 + revision: 1355db6a052410cfd62085d94b58866fd0f2c3c5 + - repository: nvidia/Qwen3.6-35B-A3B-NVFP4 + revision: 491c2f1ea524c639598bf8fa787a93fed5a6fbce + submission_flags: + agentic: + runtime_settings.agentic_inference.enable_salt: { equals: true } + runtime_settings.agentic_inference.inject_tool_delay: { equals: true } + runtime_settings.agentic_inference.stop_issuing_on_first_user_complete: + { equals: false } + accuracy.inline.eval_method: { equals: agentic_inference_inline } + accuracy.swe_bench.eval_method: { equals: swe_bench_scorer } + accuracy.swe_bench.num_repeats: { equals: 1 } + swebench_templates: + { + qwen3_6-35b-a3b: { equals: qwen_tools }, + kimi-k3: { absent: true }, + deepseek-v4_1-flash: { absent: true }, + } + disclosure: + required_fields: + [ + division, + max_supported_concurrency, + model_name, + model_precision, + link_to_model, + dataset_name, + dataset_type, + dataset_link, + shared_src, + shared_docs, + seed_set, + target_cohort, + ] + steady_state: + statuses: + [windowable, insufficient_duration, insufficient_passes, partial_dataset] + verdicts: [steady_state, drifting_up, drifting_down, anomaly, not_found] + metric_states: [plateau, drifting_up, drifting_down] + minimum_super_passes: 4 + minimum_duration_ms: + { + low_latency: 600000, + low_concurrency: 1200000, + med_concurrency: 1200000, + high_concurrency: 1200000, + margin: 1200000, + } + regions: + ultra_low_maximum: 32 + logarithmic_partitions: 3 + rounding: half_even + margin_multiplier: 1.1 + margin_rounding: ceiling + margin_satisfies_high_coverage: false + power: + cooling_overhead: { liquid: 0.3, air: 0.5, mixed: 0.5 } + computed_tolerance_w: 0.5 + computed_tolerance_kw: 0.005 + defaults: + cpu: + - architecture: x86 + maximum_cores: 64 + watts: 350 + - architecture: x86 + minimum_cores: 65 + watts: 500 + - architecture: arm + maximum_cores: 128 + watts: 300 + accelerators: + { + gb300: 1400, + gb200: 1200, + b300: 1100, + b200: 1000, + mi455x: 2500, + mi355x: 1400, + mi350x: 1000, + tpuv7: 1000, + tn3: 700, + } + switches: + sn6810ld: { passive: 1960, active_optical: 1960 } + sn5610: { passive: 900, active_optical: 2080 } + sn5400: { passive: 670 } + sn4700: { passive: 630 } + nic_watts: 75 + metrics: + { + relative_tolerance: 0.01, + relative_denominator_floor: 1.0e-09, + utilization_absolute_tolerance: 0.1, + } +enrollment: + models: + [ + llama3_1-8b, + gpt-oss-120b, + deepseek-r1, + kimi-k3, + qwen3_6-35b-a3b, + deepseek-v4_1-flash, + ] + check_groups: [submission, system, curve, point, diagnostics] + overrides: [] +diagnostics: + scope: validator + severity: warning + checks: + seed-set-registry: + { + kind: catalog_integrity, + catalog: context.cohorts.seed_sets, + on_unavailable: block_dependent_checks, + } + drafter-list-registry: + { + kind: catalog_integrity, + catalog: catalogs.approved_sped_decode_heads, + on_unavailable: block_dependent_checks, + } diff --git a/src/inference_endpoint/validation/policies/2026-10-C1/curve_checks.yaml b/src/inference_endpoint/validation/policies/2026-10-C1/curve_checks.yaml new file mode 100644 index 000000000..c5b1b8c91 --- /dev/null +++ b/src/inference_endpoint/validation/policies/2026-10-C1/curve_checks.yaml @@ -0,0 +1,217 @@ +# Cohort policy; all five files must pin the same version and revision. +version: 2026-10-C1 +revision: 1 +scope: curve +severity: error +checks: + point-dirs: + { + kind: presence, + target: curve.point_directories, + pattern: "^r[0-9]+$", + minimum_count: 1, + } + measurement-points-present: + { kind: presence, target: curve.parsed_points, minimum_count: 1 } + system-description-consistency: + kind: consistency + target: point.system_description + exclude_fields: + [ + tps_utilization, + tensor_parallel, + pipeline_parallel, + expert_parallel, + data_parallel, + disaggregated, + batch, + config_summary, + config_summary_notes, + ] + comparison: whole_document + max-concurrency-declared: + { kind: report, target: curve.max_supported_concurrency, severity: info } + region-basis: + kind: region_basis + source: curve.parsed_points + minimum_concurrency: smallest_submitted + upper_clamp: 32 + on_no_parsed_points: error + on_partial_parse: warning + region-computation: + { + kind: region_boundaries, + constants: catalogs.regions, + require_c_min: [1, 32], + require_c_max_greater_than: 32, + } + model-name-valid: + kind: membership + target: point.model_name + catalog: enrollment.models + matching: exact + skip_missing: true + deduplicate: declared_name + model-name-consistency: + kind: consistency + left: curve.model_directory_name + right: curve.first_declared_model_name + comparison: exact + on_missing: warning + config-consistency-model: + { kind: consistency, target: point.model_name, comparison: exact } + config-consistency-dataset: + { kind: consistency, target: point.dataset, comparison: exact } + benchmark-type-consistency: + kind: consistency + target: point.runtime_settings.load_pattern + unless: { offline: [dedicated] } + comparison: exact + on_disagreement_curve_type: single_turn + point-count: + kind: collection_size + source: curve.point_directories + minimum: 7 + overrides: + - when: + curve_type: single_turn + has_offline: dedicated + minimum: 8 + point-cap: + { kind: collection_size, source: curve.point_directories, maximum: 32 } + ultra-low-concurrency-coverage: + { + kind: coverage, + target: curve.parsed_points, + band: { minimum: 1, maximum: 32 }, + minimum_count: 1, + } + low-concurrency-coverage: + { + kind: coverage, + target: curve.parsed_points, + requires: [computed_regions], + band: low_concurrency, + minimum_count: 1, + } + med-concurrency-coverage: + { + kind: coverage, + target: curve.parsed_points, + requires: [computed_regions], + band: med_concurrency, + minimum_count: 1, + } + high-concurrency-coverage: + kind: coverage + target: curve.parsed_points + requires: [computed_regions] + band: high_concurrency + minimum_count: 1 + margin_counts: false + offline-point-present: + kind: offline + operation: declaration_count + single_turn_count: 1 + agentic_count: 0 + elected_must_equal: curve.max_supported_concurrency + offline-ordering: + kind: offline + operation: ordering + applies_to: { offline: [dedicated] } + severity: warning + concurrency_floor: curve.max_supported_concurrency + throughput_reference: c_max_point + throughput_minimum_multiplier: 0.98 + on_missing_summary: skip + tps-utilization: + kind: derived_metric + operation: curve_peak_ratio + stored: point.system_description.tps_utilization + source: result_summary.raw_derived_system_tps + tolerance: { absolute: 0.1 } + peak_source: points_with_readable_throughput_and_utilization + on_missing_or_invalid_file: skip + on_nonpositive_peak: skip + seed-set-consistency: + { kind: consistency, target: point.seed_set, comparison: exact } + accuracy-coverage: + kind: accuracy_coverage + requires: [computed_regions] + mandatory_bands: catalogs.mandatory_accuracy_bands + minimum_results_per_band: 1 + single_turn_requires_offline_accuracy: true + margin_counts: false + accuracy-gate: + kind: accuracy_gate + operation: single_turn_metrics + applies_to: { load_pattern: [concurrency, max_throughput] } + models: catalogs.models + aggregation: sample_weighted_mean + required_datasets: model.accuracy.datasets + dataset_catalog: catalogs.datasets + on_missing_required_dataset: error + on_unrelated_dataset: exclude + on_missing_weights: arithmetic_mean + metric_matching: case_insensitive + on_missing_metric: skip + on_unknown_model: warning + fraction_conversion: legacy_value_and_threshold_heuristic + unnamed_scalar: sole_metric_only_with_warning + accuracy-sample-count: + kind: accuracy_gate + operation: issued_count + applies_to: { load_pattern: [concurrency, max_throughput] } + models: catalogs.models + required_datasets: model.accuracy.datasets + dataset_catalog: catalogs.datasets + threshold: sample_count + count_scope: per_required_dataset_or_suite + on_missing_required_dataset: error + on_unrelated_dataset: exclude + units: samples_times_repeats + missing_repeats: 1 + minimum_repeats: 1 + on_no_sample_counts: skip + agentic-accuracy: + kind: accuracy_gate + operation: agentic_profile_available + applies_to: { load_pattern: [agentic_inference] } + models: catalogs.models + on_unknown_or_unpublished: warning + agentic-accuracy-inline: + kind: accuracy_gate + operation: per_point_fraction + applies_to: { load_pattern: [agentic_inference] } + models: catalogs.models + dataset: agentic_combined + score: score_or_first_metric + input_range: [0, 1] + output_scale: 100 + minimum: model.accuracy.inline_minimum_percent + on_missing: warning + agentic-accuracy-swebench: + kind: accuracy_gate + operation: mean_of_band_means + applies_to: { load_pattern: [agentic_inference] } + models: catalogs.models + dataset: swe_bench + score: score_or_first_metric + input_range: [0, 1] + output_scale: 100 + bands: catalogs.mandatory_accuracy_bands + required_band_count: 4 + minimum: model.accuracy.swebench_mean_minimum_percent + on_multiple_results_per_band: average_and_warn + on_missing_bands: gate_available_mean_and_warn + on_no_results: warning + agentic-osl-range: + kind: accuracy_gate + operation: per_point_range + applies_to: { load_pattern: [agentic_inference] } + models: catalogs.models + source: result_summary.output_sequence_lengths_full_run.output_sequence_lengths.avg + bounds: model.accuracy.full_run_osl_range + inclusive: true + on_missing: warning + windowed_fallback: false diff --git a/src/inference_endpoint/validation/policies/2026-10-C1/point_checks.yaml b/src/inference_endpoint/validation/policies/2026-10-C1/point_checks.yaml new file mode 100644 index 000000000..044d23488 --- /dev/null +++ b/src/inference_endpoint/validation/policies/2026-10-C1/point_checks.yaml @@ -0,0 +1,346 @@ +# Cohort policy; all five files must pin the same version and revision. +version: 2026-10-C1 +revision: 1 +scope: point +severity: error +checks: + submission-flags: + kind: field_constraints + applies_to: { load_pattern: [agentic_inference] } + constraints: catalogs.submission_flags.agentic + on_missing: error + requires: [point_config] + swebench-template: + kind: field_constraints + applies_to: { load_pattern: [agentic_inference] } + target: accuracy.swe_bench.extras.swebench_template + constraints_by_model: catalogs.submission_flags.swebench_templates + requires: [point_config] + agentic-trajectory-count: + kind: dataset_count + applies_to: { load_pattern: [agentic_inference] } + dataset: agentic_combined + source: point.runtime_settings.agentic_inference.num_trajectories_to_issue + sample_unit: trajectory + operator: positive_multiple + catalog: catalogs.datasets + on_missing: error + requires: [point_config] + swebench-instance-count: + kind: dataset_count + applies_to: { load_pattern: [agentic_inference] } + dataset: swe_bench + sources: + [ + accuracy.swe_bench.extras.num_instances, + accuracy.swe_bench.evaluated_instance_count, + ] + sample_unit: instance + operator: equal + catalog: catalogs.datasets + require_all: true + requires: [point_config] + endpoints-client-sha: + kind: artifact_binding + source: result_summary.git_sha + format: git_commit_sha1 + catalog: catalogs.approved_client_revisions + on_missing: error + on_empty_approval_list: blocked + matching: exact_revision + repository: https://github.com/mlcommons/endpoints + approved-checkpoint: + kind: artifact_binding + sources: [point.checkpoint.repository, point.checkpoint.revision] + catalog: catalogs.approved_checkpoints + matching: model_and_repository_and_revision + on_missing: error + on_empty_model_catalog: blocked + point-power: + kind: power + operation: engaged_nodes + requires: [power_computation, point_config] + source: point.nodes_used + on_undeclared_nodes: fully_engaged + declared_total_scaling: maximum_engaged_fraction + switch_scaling: engaged_node_fraction + round_kw_decimals: 2 + point-config-valid: + { kind: artifact_schema, artifact: point.yaml, schema: point_config } + point-dirname-concurrency: + kind: consistency + left: point.directory_concurrency + right: point.concurrency + comparison: exact + requires: [point_config] + system-description-present: + { kind: presence, target: point.system_desc.json, object: file } + system-description-valid: + { + kind: artifact_schema, + artifact: system_desc.json, + schema: system_description, + } + result-summary-present: + { kind: presence, target: point.result_summary.json, object: file } + result-file-valid: + { + kind: artifact_schema, + artifact: result_summary.json, + schema: result_summary, + } + accuracy-valid: + kind: artifact_schema + accept: [accuracy_results.json, results.json.accuracy_scores] + schema: accuracy_result + when_present: true + precedence: standalone_first + reject_empty: true + point-disclosure-complete: + { kind: disclosure, fields: catalogs.disclosure.required_fields } + target-cohort: + { + kind: cohort_identifier, + target: point.target_cohort, + format: year_month_c0_or_c1, + requires: [point_config], + } + shared-path-resolution: + kind: path_resolution + targets: [point.shared_src, point.shared_docs] + base: submission.root + require_directory: true + allow_absolute: false + allow_parent_traversal: false + allow_escape_via_symlink: false + on_missing: skip + offline-declared: + { + kind: membership, + target: point.offline, + catalog: catalogs.offline_modes, + required: false, + requires: [point_config], + } + region-declared: + { + kind: membership, + target: point.region, + catalog: catalogs.declared_regions, + required: false, + requires: [point_config], + } + concurrency-in-range: + kind: region_placement + operation: in_range + requires: [computed_regions, point_config] + include_margin: true + unless: { offline: [dedicated] } + region-placement: + kind: region_placement + operation: declared_matches_computed + requires: [computed_regions] + severity: warning + ignore_declared: [submitters_choice] + on_missing_or_unclassifiable: skip + load-pattern: + kind: issuance + allowed: [concurrency, agentic_inference] + require_positive_concurrency: true + unless: { offline: [dedicated] } + requires: [point_config] + streaming-config: + { + kind: report, + target: point.runtime_settings.stream_all_chunks, + severity: info, + interpretation: client_ipc_forwarding_only, + } + warmup-present: { kind: presence, target: point.warmup, object: declaration } + warmup-logs-retained: + kind: warmup + operation: log_retention + applies_to: { declared: [warmup] } + requires: [warmup, point_config] + severity: warning + disabled_warmup: exempt + require_declared_retained: true + verify_archive_contents: false + warmup-salt: + kind: warmup + operation: salt_disabled + source: point.runtime_settings.warmup.salt + severity: warning + on_missing: skip + requires: [point_config] + steady-state-valid: + kind: steady_state + operation: vocabulary + applies_to: { declared: [steady_state] } + requires: [steady_state, point_config] + constants: catalogs.steady_state + steady-state-consistency: + kind: steady_state + operation: status_window_agreement + applies_to: { declared: [steady_state] } + requires: [steady_state, point_config] + constants: catalogs.steady_state + official_requires_plateau: true + check_reported_super_pass_count: true + on_missing_window_extent: skip_count_comparison + steady-state-basis: + kind: steady_state + operation: reporting_basis + constants: catalogs.steady_state + on_missing_or_nonofficial: warning + fallback: whole_run_total + on_drift: warn_range_or_slope_required + point-duration: + kind: duration + requires: [computed_regions] + severity: error + thresholds: catalogs.steady_state.minimum_duration_ms + basis_precedence: [reported_window_issue_span, whole_run_duration] + window_status_required: false + on_unclassifiable_concurrency: skip + min-query-count: + kind: dataset_minimum + source: result_summary.n_samples_completed + catalog: catalogs.datasets + threshold: sample_count + supported_sample_units: [sample] + on_unsupported_sample_unit: skip + on_unknown_or_null_threshold: skip + metric-consistency-duration: + kind: comparison + left: { kind: field, field: result_summary.duration_ns } + operator: greater_than + right: { kind: constant, value: 0 } + metric-consistency-accounting: + kind: comparison + when: { field: result_summary.n_samples_issued, greater_than: 0 } + left: { kind: field, field: result_summary.n_samples_completed } + operator: equal + right: { kind: field, field: result_summary.n_samples_issued } + metric-consistency-output-tokens: + kind: comparison + left: { kind: field, field: result_summary.output_sequence_lengths.total } + operator: greater_than_or_equal + right: { kind: constant, value: 0 } + metric-consistency-system-tps: + kind: derived_metric + operation: output_tokens_per_elapsed_second + stored: result_summary.system_tps + constants: catalogs.metrics + on_missing_stored: report_derived + metric-consistency-tpot-p90: + kind: numeric_validity + source: result_summary.tpot.percentiles.90 + require_present: true + finite: true + strictly_positive: true + verify_underlying_distribution: false + metric-consistency-tps-per-user: + kind: derived_metric + operation: inverse_tpot_p90_ms + numerator: 1000 + stored: result_summary.tps_per_user + constants: catalogs.metrics + on_invalid_tpot: skip + on_missing_stored: report_derived + metric-consistency-tps-per-kw: + kind: derived_metric + operation: throughput_per_provisioned_kw + denominator: point.power_kw + stored: result_summary.system_tps_per_kw + constants: catalogs.metrics + on_missing_or_nonpositive_power: skip + on_missing_stored: report_derived + requires: [point_power, point_config] + agentic-metric-consistency: + kind: derived_metric + operation: output_tokens_per_completed_turn_second + stored: result_summary.e2e_avg_interactivity + constants: catalogs.metrics + on_stored_without_derivable_inputs: error + on_no_inputs_and_no_stored: skip + on_missing_stored: report_derived + seed-config-legacy: + kind: seed_binding + operation: legacy_names + aliases: + { + scheduler_random_seed: scheduler_rng_seed, + dataloader_random_seed: sample_index_rng_seed, + } + severity: warning + warn_legacy_only_when_model_seed_missing: true + seed-set-membership: + kind: seed_binding + operation: membership + catalog: context.cohorts.seed_sets + on_unavailable: blocked + requires: [point_config, seed_catalog] + seed-runtime-match: + kind: seed_binding + operation: runtime_values + catalog: context.cohorts.seed_sets + on_unavailable: blocked + fields: [scheduler_rng_seed, sample_index_rng_seed, model_seed] + require_all: true + requires: [point_config, seed_catalog] + seed-set-adoption: + kind: seed_binding + operation: adoption_window + catalog: context.cohorts.seed_sets + on_unavailable: blocked + adoption_window_cohorts: 4 + on_legacy_registry_without_cohorts: blocked + requires: [point_config, seed_catalog] + approved-drafter: + kind: drafter_binding + operation: membership + applies_to: { speculative_decoding: true } + requires: [approved_sped_decode_heads, point_config] + catalog: catalogs.approved_sped_decode_heads + matching: model_and_repository_and_revision + on_missing_repository_or_revision: error + on_empty_catalog: error + drafter-approval-lead-time: + kind: drafter_binding + operation: approval_age + applies_to: { speculative_decoding: true } + requires: [approved_sped_decode_heads, point_config] + catalog: catalogs.approved_sped_decode_heads + minimum_cohorts: 2 + on_unmatched_drafter: skip + on_unparseable_cohort: warning + on_missing_approval_cohort: warning + nodes-used: + kind: power + operation: node_declaration + requires: [point_config, power_computation] + source: point.nodes_used + reject_duplicate_ensemble_ids: true + require_known_ensemble_ids: true + require_nodes_within_provisioned: true + require_accelerator_capacity: true + on_spare_nodes: warning + on_undeclared_nodes: fully_engaged + maximal-engagement: + kind: power + operation: maximal_engagement + requires: [point_config, power_computation] + parallelism: point.system_description + replica_factors: [tensor_parallel, pipeline_parallel, expert_parallel] + require_maximum_data_parallel: true + shortfall: point.dp_shortfall + on_missing_parallelism: warning + on_disaggregated: warning + on_unknown_accelerator_count: warning + point-rules-skipped: + kind: report + severity: warning + applies_to: { invalid: [point_config] } + target: point.directory + operation: blocked_config_dependents diff --git a/src/inference_endpoint/validation/policies/2026-10-C1/submission_checks.yaml b/src/inference_endpoint/validation/policies/2026-10-C1/submission_checks.yaml new file mode 100644 index 000000000..29f2e9b8d --- /dev/null +++ b/src/inference_endpoint/validation/policies/2026-10-C1/submission_checks.yaml @@ -0,0 +1,36 @@ +# Cohort policy; all five files must pin the same version and revision. +version: 2026-10-C1 +revision: 1 +scope: submission +severity: error +checks: + path-exists: { kind: presence, target: submission.root, object: path } + required-dir: { kind: presence, targets: [results, docs], object: directory } + system-results-dir: + { kind: presence, target: submission.system_directories, minimum_count: 1 } + src-dir: + { + kind: presence, + target: submission.implementation_directories, + minimum_count: 1, + } + src-readme: + { + kind: presence, + target: implementation.readme, + scope: implementation, + case_sensitive: false, + } + accuracy-present: + { + kind: accuracy_presence, + minimum_models: 1, + accept: [standalone_accuracy_results, embedded_nonempty_accuracy_scores], + } + system-name-consistency: + kind: consistency + target: point.system_description.system_name + comparison: exact + require_nonempty: true + required_artifact: point.system_desc.json + on_missing: error diff --git a/src/inference_endpoint/validation/policies/2026-10-C1/system_checks.yaml b/src/inference_endpoint/validation/policies/2026-10-C1/system_checks.yaml new file mode 100644 index 000000000..718b314d8 --- /dev/null +++ b/src/inference_endpoint/validation/policies/2026-10-C1/system_checks.yaml @@ -0,0 +1,46 @@ +# Cohort policy; all five files must pin the same version and revision. +version: 2026-10-C1 +revision: 1 +scope: system +severity: error +checks: + benchmark-model-dir: + { kind: presence, target: system.model_directories, minimum_count: 1 } + power-descriptor: + kind: power + operation: descriptor + artifact: system_power.json + schema: appendix_e_system_power + constants: catalogs.power + unless: { division: [RDI, Serviced], not_declared: [power_descriptor] } + require_derivable_total: true + validate_public_sources: true + validate_node_sets: true + validate_scale_out: true + validate_computed_values: true + cooling_must_match_system_description: true + power-estimated: + { + kind: power, + operation: appendix_d_defaults, + constants: catalogs.power, + severity: warning, + tag: estimated_power, + } + system-description-across-curves: + kind: consistency + target: point.system_description + scope: system + exclude_fields: + [ + tps_utilization, + tensor_parallel, + pipeline_parallel, + expert_parallel, + data_parallel, + disaggregated, + batch, + config_summary, + config_summary_notes, + ] + comparison: whole_document diff --git a/src/inference_endpoint/validation/references.py b/src/inference_endpoint/validation/references.py new file mode 100644 index 000000000..5875c7ff0 --- /dev/null +++ b/src/inference_endpoint/validation/references.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Validate explicit evidence addresses and references to policy catalog entries.""" + +import re +from collections.abc import Mapping + +from .vocabulary import EvidenceReference + + +def evidence_references(value: object) -> frozenset[EvidenceReference]: + found: set[EvidenceReference] = set() + if isinstance(value, Mapping): + for child in value.values(): + found.update(evidence_references(child)) + elif isinstance(value, (list, tuple)): + for child in value: + found.update(evidence_references(child)) + elif isinstance(value, str) and re.match( + r"^(point|curve|system|submission|result_summary|accuracy|implementation|model)\.", + value, + ): + found.add(EvidenceReference(value)) + return frozenset(found) + + +def validate_catalog_references(value: object, catalogs: dict, enrollment: dict): + if isinstance(value, dict): + for child in value.values(): + validate_catalog_references(child, catalogs, enrollment) + elif isinstance(value, list): + for child in value: + validate_catalog_references(child, catalogs, enrollment) + elif isinstance(value, str): + if value.startswith("catalogs.") or value.startswith("enrollment."): + root, *components = value.split(".") + cursor = catalogs if root == "catalogs" else enrollment + for component in components: + if not isinstance(cursor, dict) or component not in cursor: + raise ValueError(f"Unresolved policy reference {value}") + cursor = cursor[component] + elif value.startswith("context.") and value != "context.cohorts.seed_sets": + raise ValueError(f"Unknown external context reference {value}") + + +def needs_model(requirements: Mapping) -> bool: + """Recognize the supported parameter bindings that need model enrollment.""" + return ( + any( + reference.value.startswith("model.") + for reference in evidence_references(requirements) + ) + or requirements.get("models") == "catalogs.models" + or "constraints_by_model" in requirements + or requirements.get("matching") == "model_and_repository_and_revision" + ) diff --git a/src/inference_endpoint/validation/schemas/__init__.py b/src/inference_endpoint/validation/schemas/__init__.py new file mode 100644 index 000000000..c332fe567 --- /dev/null +++ b/src/inference_endpoint/validation/schemas/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy parsers selected by cohort and inclusive revision intervals.""" + +from .registry import BundleParser, parser_for +from .v1 import ParserV1 + +__all__ = ["BundleParser", "ParserV1", "parser_for"] diff --git a/src/inference_endpoint/validation/schemas/registry.py b/src/inference_endpoint/validation/schemas/registry.py new file mode 100644 index 000000000..e3fec6b43 --- /dev/null +++ b/src/inference_endpoint/validation/schemas/registry.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Explicit cohort and inclusive revision-range parser selection.""" + +from abc import ABC, abstractmethod +from typing import ClassVar + +from ..types import Release + + +class BundleParser(ABC): + registry: ClassVar[list[tuple[str, int, int | None, type["BundleParser"]]]] = [] + + def __init_subclass__( + cls, *, version: str, first_revision: int, last_revision: int | None, **kwargs + ): + super().__init_subclass__(**kwargs) + Release(version=version, revision=first_revision) + if last_revision is not None and type(last_revision) is not int: + raise ValueError("Parser revision boundaries must be integers") + if last_revision is not None and last_revision < first_revision: + raise ValueError("Invalid parser revision interval") + for known_version, first, last, _ in cls.registry: + if known_version == version and max(first, first_revision) <= min( + last if last is not None else float("inf"), + last_revision if last_revision is not None else float("inf"), + ): + raise ValueError("Overlapping parser revision intervals") + cls.registry.append((version, first_revision, last_revision, cls)) + + @abstractmethod + def __call__(self, documents, digests): + """Normalize a complete document set into an immutable policy.""" + + +def parser_for(release: Release) -> BundleParser: + matches = [ + parser + for version, first, last, parser in BundleParser.registry + if version == release.version + and first <= release.revision + and (last is None or release.revision <= last) + ] + if len(matches) != 1: + raise ValueError( + f"Unsupported policy release {release.version} revision {release.revision}" + ) + return matches[0]() diff --git a/src/inference_endpoint/validation/schemas/requirements_v1.py b/src/inference_endpoint/validation/schemas/requirements_v1.py new file mode 100644 index 000000000..c47d8fb73 --- /dev/null +++ b/src/inference_endpoint/validation/schemas/requirements_v1.py @@ -0,0 +1,610 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Strict check requirement contracts for cohort revision 1.""" + +from collections.abc import Mapping +from typing import Annotated, Literal, Self + +from pydantic import Field, StrictBool, StrictFloat, StrictInt, model_validator + +from inference_endpoint.config.schema import LoadPatternType + +from ..operations import ( + AccuracyOperation, + BindingMatch, + ComparisonOperator, + DatasetCountOperator, + DerivedOperation, + DrafterOperation, + DurationBasis, + OfflineOperation, + OperandKind, + PowerOperation, + RegionPlacementOperation, + SeedOperation, + SteadyOperation, + WarmupOperation, +) +from ..types import AccuracyKind, FrozenModel, Identifier, OfflineMode +from ..vocabulary import CheckKind + +Number = Annotated[StrictInt | StrictFloat, Field(allow_inf_nan=False)] +NonnegativeNumber = Annotated[Number, Field(ge=0)] +PositiveNumber = Annotated[Number, Field(gt=0)] +NonnegativeInt = Annotated[StrictInt, Field(ge=0)] +PositiveInt = Annotated[StrictInt, Field(gt=0)] + + +class Requirements(FrozenModel): + @model_validator(mode="before") + @classmethod + def reject_null_fields(cls, value: object) -> object: + if isinstance(value, Mapping): + nulls = [name for name, child in value.items() if child is None] + if nulls: + raise ValueError( + f"Requirement fields cannot be null: {', '.join(nulls)}" + ) + return value + + def require_fields(self, *names: str) -> None: + missing = [name for name in names if name not in self.model_fields_set] + if missing: + raise ValueError(f"Required fields: {', '.join(missing)}") + + def require_exactly_one(self, left: str, right: str) -> None: + if (left in self.model_fields_set) == (right in self.model_fields_set): + raise ValueError(f"Requires exactly one of {left}, {right}") + + @model_validator(mode="after") + def nonempty_sequences(self) -> Self: + for name in ( + "accept", + "sources", + "targets", + "allowed", + "fields", + "basis_precedence", + "replica_factors", + ): + if name in self.model_fields_set and not getattr(self, name): + raise ValueError(f"{name} must not be empty") + return self + + +class Bounds(Requirements): + minimum: NonnegativeInt + maximum: NonnegativeInt + + @model_validator(mode="after") + def ordered(self): + if self.minimum > self.maximum: + raise ValueError("minimum must not exceed maximum") + return self + + +class Tolerance(Requirements): + absolute: NonnegativeNumber + + +class ComparisonWhen(Requirements): + field: Identifier + greater_than: Number + + +class FieldOperand(Requirements): + kind: Literal[OperandKind.FIELD] + field: Identifier + + +class ConstantOperand(Requirements): + kind: Literal[OperandKind.CONSTANT] + value: Number + + +Operand = Annotated[FieldOperand | ConstantOperand, Field(discriminator="kind")] + + +class CountWhen(Requirements): + curve_type: Literal[AccuracyKind.SINGLE_TURN] + has_offline: Literal[OfflineMode.DEDICATED, OfflineMode.ELECTED] + + +class CountOverride(Requirements): + when: CountWhen + minimum: NonnegativeInt + + +class AccuracyCoverageRequirements(Requirements): + mandatory_bands: Identifier + margin_counts: StrictBool | None = None + minimum_results_per_band: NonnegativeInt + single_turn_requires_offline_accuracy: StrictBool | None = None + + +class AccuracyGateRequirements(Requirements): + aggregation: Identifier | None = None + bands: Identifier | None = None + bounds: Identifier | None = None + count_scope: Identifier | None = None + dataset: Identifier | None = None + dataset_catalog: Identifier | None = None + fraction_conversion: Identifier | None = None + inclusive: StrictBool | None = None + input_range: tuple[Number, Number] | None = None + metric_matching: Identifier | None = None + minimum: NonnegativeNumber | Identifier | None = None + minimum_repeats: PositiveInt | None = None + missing_repeats: PositiveInt | None = None + models: Identifier + on_missing: Identifier | None = None + on_missing_bands: Identifier | None = None + on_missing_metric: Identifier | None = None + on_missing_required_dataset: Identifier | None = None + on_missing_weights: Identifier | None = None + on_multiple_results_per_band: Identifier | None = None + on_no_results: Identifier | None = None + on_no_sample_counts: Identifier | None = None + on_unknown_model: Identifier | None = None + on_unknown_or_unpublished: Identifier | None = None + on_unrelated_dataset: Identifier | None = None + operation: AccuracyOperation + output_scale: PositiveNumber | None = None + required_band_count: PositiveInt | None = None + required_datasets: Identifier | None = None + score: Identifier | None = None + source: Identifier | None = None + threshold: Identifier | None = None + units: Identifier | None = None + unnamed_scalar: Identifier | None = None + windowed_fallback: StrictBool | None = None + + @model_validator(mode="after") + def check_form(self) -> Self: + if self.operation in ( + AccuracyOperation.SINGLE_TURN_METRICS, + AccuracyOperation.ISSUED_COUNT, + ): + self.require_fields("required_datasets", "dataset_catalog") + if self.operation is AccuracyOperation.ISSUED_COUNT: + self.require_fields("threshold", "missing_repeats", "minimum_repeats") + elif self.operation in ( + AccuracyOperation.PER_POINT_FRACTION, + AccuracyOperation.MEAN_OF_BAND_MEANS, + ): + self.require_fields("minimum", "dataset", "input_range", "output_scale") + assert self.input_range is not None + if self.input_range[0] > self.input_range[1]: + raise ValueError("input_range must be ordered") + if self.operation is AccuracyOperation.MEAN_OF_BAND_MEANS: + self.require_fields("bands", "required_band_count") + elif self.operation is AccuracyOperation.PER_POINT_RANGE: + self.require_fields("source", "bounds") + return self + + +class AccuracyPresenceRequirements(Requirements): + accept: list[Identifier] + minimum_models: NonnegativeInt + + +class ArtifactBindingRequirements(Requirements): + catalog: Identifier + format: Identifier | None = None + matching: BindingMatch + on_empty_approval_list: Identifier | None = None + on_empty_model_catalog: Identifier | None = None + on_missing: Identifier | None = None + repository: Identifier | None = None + source: Identifier | None = None + sources: list[Identifier] | None = None + + @model_validator(mode="after") + def check_form(self) -> Self: + self.require_fields( + "sources" if self.matching is BindingMatch.CHECKPOINT else "source" + ) + if self.matching is BindingMatch.CHECKPOINT: + assert self.sources is not None + if len(self.sources) != 2: + raise ValueError( + "Checkpoint sources must contain repository and revision" + ) + return self + + +class ArtifactSchemaRequirements(Requirements): + accept: list[Identifier] | None = None + artifact: Identifier | None = None + precedence: Identifier | None = None + reject_empty: StrictBool | None = None + artifact_schema: Identifier = Field(alias="schema") + when_present: StrictBool | None = None + + +class CatalogIntegrityRequirements(Requirements): + catalog: Identifier + on_unavailable: Identifier | None = None + + +class CohortIdentifierRequirements(Requirements): + format: Identifier + target: Identifier + + +class CollectionSizeRequirements(Requirements): + maximum: NonnegativeInt | None = None + minimum: NonnegativeInt | None = None + overrides: list[CountOverride] | None = None + source: Identifier + + @model_validator(mode="after") + def check_form(self) -> Self: + if self.minimum is None and self.maximum is None: + raise ValueError("Requires minimum or maximum") + if ( + self.minimum is not None + and self.maximum is not None + and self.minimum > self.maximum + ): + raise ValueError("minimum exceeds maximum") + return self + + +class ComparisonRequirements(Requirements): + left: Operand + operator: ComparisonOperator + right: Operand + when: ComparisonWhen | None = None + + +class ConsistencyRequirements(Requirements): + comparison: Identifier | None = None + exclude_fields: list[Identifier] | None = None + left: Identifier | None = None + on_disagreement_curve_type: Identifier | None = None + on_missing: Identifier | None = None + require_nonempty: StrictBool | None = None + required_artifact: Identifier | None = None + right: Identifier | None = None + target: Identifier | None = None + + @model_validator(mode="after") + def check_form(self) -> Self: + self.require_exactly_one("target", "left") + if "left" in self.model_fields_set: + self.require_fields("right") + elif "right" in self.model_fields_set: + raise ValueError("right requires left") + return self + + +class CoverageRequirements(Requirements): + band: Identifier | Bounds + margin_counts: StrictBool | None = None + minimum_count: NonnegativeInt + target: Identifier + + +class DatasetCountRequirements(Requirements): + catalog: Identifier + dataset: Identifier + on_missing: Identifier | None = None + operator: DatasetCountOperator + require_all: StrictBool | None = None + sample_unit: Identifier | None = None + source: Identifier | None = None + sources: list[Identifier] | None = None + + @model_validator(mode="after") + def check_form(self) -> Self: + self.require_exactly_one("source", "sources") + return self + + +class DatasetMinimumRequirements(Requirements): + catalog: Identifier + on_unknown_or_null_threshold: Identifier | None = None + on_unsupported_sample_unit: Identifier | None = None + source: Identifier + supported_sample_units: list[Identifier] | None = None + threshold: Identifier + + +class DerivedMetricRequirements(Requirements): + constants: Identifier | None = None + denominator: Identifier | None = None + numerator: PositiveNumber | None = None + on_invalid_tpot: Identifier | None = None + on_missing_or_invalid_file: Identifier | None = None + on_missing_or_nonpositive_power: Identifier | None = None + on_missing_stored: Identifier | None = None + on_no_inputs_and_no_stored: Identifier | None = None + on_nonpositive_peak: Identifier | None = None + on_stored_without_derivable_inputs: Identifier | None = None + operation: DerivedOperation + peak_source: Identifier | None = None + source: Identifier | None = None + stored: Identifier + tolerance: Tolerance | None = None + + @model_validator(mode="after") + def check_form(self) -> Self: + if self.operation is DerivedOperation.CURVE_PEAK_RATIO: + self.require_fields("tolerance") + elif self.operation is DerivedOperation.INVERSE_TPOT_P90_MS: + self.require_fields("numerator") + elif self.operation is DerivedOperation.THROUGHPUT_PER_PROVISIONED_KW: + self.require_fields("denominator") + return self + + +class DisclosureRequirements(Requirements): + fields: Identifier + + +class DrafterBindingRequirements(Requirements): + catalog: Identifier + matching: BindingMatch | None = None + minimum_cohorts: PositiveInt | None = None + on_empty_catalog: Identifier | None = None + on_missing_approval_cohort: Identifier | None = None + on_missing_repository_or_revision: Identifier | None = None + on_unmatched_drafter: Identifier | None = None + on_unparseable_cohort: Identifier | None = None + operation: DrafterOperation + + @model_validator(mode="after") + def check_form(self) -> Self: + if self.operation is DrafterOperation.APPROVAL_AGE: + self.require_fields("minimum_cohorts") + return self + + +class DurationRequirements(Requirements): + basis_precedence: list[DurationBasis] + on_unclassifiable_concurrency: Identifier | None = None + thresholds: Identifier + window_status_required: StrictBool | None = None + + +class FieldConstraintsRequirements(Requirements): + constraints: Identifier | None = None + constraints_by_model: Identifier | None = None + on_missing: Identifier | None = None + target: Identifier | None = None + + @model_validator(mode="after") + def check_form(self) -> Self: + self.require_exactly_one("constraints", "constraints_by_model") + if self.constraints_by_model is not None: + self.require_fields("target") + return self + + +class IssuanceRequirements(Requirements): + allowed: list[LoadPatternType] + require_positive_concurrency: StrictBool | None = None + + +class MembershipRequirements(Requirements): + catalog: Identifier + deduplicate: StrictBool | Identifier | None = None + matching: Identifier | None = None + required: StrictBool | None = None + skip_missing: StrictBool | None = None + target: Identifier + + +class NumericValidityRequirements(Requirements): + finite: StrictBool | None = None + require_present: StrictBool | None = None + source: Identifier + strictly_positive: StrictBool | None = None + verify_underlying_distribution: StrictBool | None = None + + +class OfflineRequirements(Requirements): + agentic_count: NonnegativeInt | None = None + concurrency_floor: NonnegativeNumber | Identifier | None = None + elected_must_equal: PositiveInt | Identifier | None = None + on_missing_summary: Identifier | None = None + operation: OfflineOperation + single_turn_count: NonnegativeInt | None = None + throughput_minimum_multiplier: NonnegativeNumber | None = None + throughput_reference: Identifier | None = None + + @model_validator(mode="after") + def check_form(self) -> Self: + if self.operation is OfflineOperation.DECLARATION_COUNT: + self.require_fields( + "agentic_count", "single_turn_count", "elected_must_equal" + ) + else: + self.require_fields("concurrency_floor", "throughput_minimum_multiplier") + return self + + +class PathResolutionRequirements(Requirements): + allow_absolute: StrictBool | None = None + allow_escape_via_symlink: StrictBool | None = None + allow_parent_traversal: StrictBool | None = None + base: Identifier + on_missing: Identifier | None = None + require_directory: StrictBool | None = None + targets: list[Identifier] + + +class PowerRequirements(Requirements): + artifact: Identifier | None = None + constants: Identifier | None = None + cooling_must_match_system_description: StrictBool | None = None + declared_total_scaling: Identifier | None = None + on_disaggregated: Identifier | None = None + on_missing_parallelism: Identifier | None = None + on_spare_nodes: Identifier | None = None + on_undeclared_nodes: Identifier | None = None + on_unknown_accelerator_count: Identifier | None = None + operation: PowerOperation + parallelism: Identifier | None = None + reject_duplicate_ensemble_ids: StrictBool | None = None + replica_factors: list[Identifier] | None = None + require_accelerator_capacity: StrictBool | None = None + require_derivable_total: StrictBool | None = None + require_known_ensemble_ids: StrictBool | None = None + require_maximum_data_parallel: StrictBool | None = None + require_nodes_within_provisioned: StrictBool | None = None + round_kw_decimals: NonnegativeInt | None = None + artifact_schema: Identifier | None = Field(default=None, alias="schema") + shortfall: Identifier | None = None + source: Identifier | None = None + switch_scaling: Identifier | None = None + tag: Identifier | None = None + validate_computed_values: StrictBool | None = None + validate_node_sets: StrictBool | None = None + validate_public_sources: StrictBool | None = None + validate_scale_out: StrictBool | None = None + + @model_validator(mode="after") + def check_form(self) -> Self: + if self.operation is PowerOperation.DESCRIPTOR: + self.require_fields("artifact", "constants") + elif self.operation is PowerOperation.ENGAGED_NODES: + self.require_fields("round_kw_decimals") + elif self.operation is PowerOperation.MAXIMAL_ENGAGEMENT: + self.require_fields("replica_factors") + return self + + +class PresenceRequirements(Requirements): + case_sensitive: StrictBool | None = None + minimum_count: NonnegativeInt | None = None + object: Identifier | None = None + pattern: Identifier | None = None + target: Identifier | None = None + targets: list[Identifier] | None = None + + @model_validator(mode="after") + def check_form(self) -> Self: + self.require_exactly_one("target", "targets") + return self + + +class RegionBasisRequirements(Requirements): + minimum_concurrency: Identifier | None = None + on_no_parsed_points: Identifier | None = None + on_partial_parse: Identifier | None = None + source: Identifier + upper_clamp: PositiveInt + + +class RegionBoundariesRequirements(Requirements): + constants: Identifier + require_c_max_greater_than: NonnegativeInt + require_c_min: tuple[PositiveInt, PositiveInt] + + +class RegionPlacementRequirements(Requirements): + ignore_declared: list[Identifier] | None = None + include_margin: StrictBool | None = None + on_missing_or_unclassifiable: Identifier | None = None + operation: RegionPlacementOperation + + +class ReportRequirements(Requirements): + interpretation: Identifier | None = None + operation: Identifier | None = None + target: Identifier + + +class SeedBindingRequirements(Requirements): + adoption_window_cohorts: PositiveInt | None = None + aliases: dict[Identifier, Identifier] | None = None + catalog: Identifier | None = None + fields: list[Identifier] | None = None + on_legacy_registry_without_cohorts: Identifier | None = None + on_unavailable: Identifier | None = None + operation: SeedOperation + require_all: StrictBool | None = None + warn_legacy_only_when_model_seed_missing: StrictBool | None = None + + @model_validator(mode="after") + def check_form(self) -> Self: + if self.operation is SeedOperation.LEGACY_NAMES: + self.require_fields("aliases") + else: + self.require_fields("catalog") + if self.operation is SeedOperation.RUNTIME_VALUES: + self.require_fields("fields") + elif self.operation is SeedOperation.ADOPTION_WINDOW: + self.require_fields("adoption_window_cohorts") + return self + + +class SteadyStateRequirements(Requirements): + check_reported_super_pass_count: StrictBool | None = None + constants: Identifier + fallback: Identifier | None = None + official_requires_plateau: StrictBool | None = None + on_drift: Identifier | None = None + on_missing_or_nonofficial: Identifier | None = None + on_missing_window_extent: Identifier | None = None + operation: SteadyOperation + + +class WarmupRequirements(Requirements): + disabled_warmup: Identifier | None = None + on_missing: Identifier | None = None + operation: WarmupOperation + require_declared_retained: StrictBool | None = None + source: Identifier | None = None + verify_archive_contents: StrictBool | None = None + + @model_validator(mode="after") + def check_form(self) -> Self: + if self.operation is WarmupOperation.SALT_DISABLED: + self.require_fields("source") + return self + + +REQUIREMENT_MODELS: dict[CheckKind, type[Requirements]] = { + CheckKind.ACCURACY_COVERAGE: AccuracyCoverageRequirements, + CheckKind.ACCURACY_GATE: AccuracyGateRequirements, + CheckKind.ACCURACY_PRESENCE: AccuracyPresenceRequirements, + CheckKind.ARTIFACT_BINDING: ArtifactBindingRequirements, + CheckKind.ARTIFACT_SCHEMA: ArtifactSchemaRequirements, + CheckKind.CATALOG_INTEGRITY: CatalogIntegrityRequirements, + CheckKind.COHORT_IDENTIFIER: CohortIdentifierRequirements, + CheckKind.COLLECTION_SIZE: CollectionSizeRequirements, + CheckKind.COMPARISON: ComparisonRequirements, + CheckKind.CONSISTENCY: ConsistencyRequirements, + CheckKind.COVERAGE: CoverageRequirements, + CheckKind.DATASET_COUNT: DatasetCountRequirements, + CheckKind.DATASET_MINIMUM: DatasetMinimumRequirements, + CheckKind.DERIVED_METRIC: DerivedMetricRequirements, + CheckKind.DISCLOSURE: DisclosureRequirements, + CheckKind.DRAFTER_BINDING: DrafterBindingRequirements, + CheckKind.DURATION: DurationRequirements, + CheckKind.FIELD_CONSTRAINTS: FieldConstraintsRequirements, + CheckKind.ISSUANCE: IssuanceRequirements, + CheckKind.MEMBERSHIP: MembershipRequirements, + CheckKind.NUMERIC_VALIDITY: NumericValidityRequirements, + CheckKind.OFFLINE: OfflineRequirements, + CheckKind.PATH_RESOLUTION: PathResolutionRequirements, + CheckKind.POWER: PowerRequirements, + CheckKind.PRESENCE: PresenceRequirements, + CheckKind.REGION_BASIS: RegionBasisRequirements, + CheckKind.REGION_BOUNDARIES: RegionBoundariesRequirements, + CheckKind.REGION_PLACEMENT: RegionPlacementRequirements, + CheckKind.REPORT: ReportRequirements, + CheckKind.SEED_BINDING: SeedBindingRequirements, + CheckKind.STEADY_STATE: SteadyStateRequirements, + CheckKind.WARMUP: WarmupRequirements, +} + + +def validate_requirements( + kind: CheckKind, parameters: Mapping[str, object] +) -> Requirements: + """Validate a complete rule or merged override using its revision's contract.""" + return REQUIREMENT_MODELS[kind].model_validate(dict(parameters)) diff --git a/src/inference_endpoint/validation/schemas/v1.py b/src/inference_endpoint/validation/schemas/v1.py new file mode 100644 index 000000000..064663dd3 --- /dev/null +++ b/src/inference_endpoint/validation/schemas/v1.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Wire models and normalization for 2026-10-C1 revision 1.""" + +import hashlib +from types import MappingProxyType + +from pydantic import Field, JsonValue, model_validator + +from ..conditions import Conditions +from ..models import Override, Policy, Rule, freeze +from ..references import evidence_references, needs_model, validate_catalog_references +from ..types import ( + EvidenceKey, + FrozenModel, + Identifier, + PolicyFile, + Release, + Scope, + Severity, +) +from ..vocabulary import CheckKind +from .registry import BundleParser +from .requirements_v1 import validate_requirements +from .values_v1 import Catalogs, Diagnostics, GroupDocument + + +class Metadata(FrozenModel): + kind: CheckKind + scope: Scope + severity: Severity + applies_to: Conditions | None = None + unless: Conditions | None = None + requires: tuple[EvidenceKey, ...] = () + + @model_validator(mode="after") + def unique_dependencies(self): + if len(self.requires) != len(set(self.requires)): + raise ValueError("Duplicate evidence dependency") + return self + + +class OverrideDocument(FrozenModel): + id: Identifier + rule: Identifier + reason: Identifier + applies_to: Conditions + enabled: bool = Field(default=True, strict=True) + requirements: dict[str, JsonValue] = Field(default_factory=dict) + + +class Enrollment(FrozenModel): + models: tuple[Identifier, ...] = Field(min_length=1) + check_groups: tuple[Identifier, ...] = Field(min_length=1) + overrides: tuple[OverrideDocument, ...] = () + + +class CatalogDocument(Release): + catalogs: Catalogs + enrollment: Enrollment + diagnostics: Diagnostics + + +class ParserV1(BundleParser, version="2026-10-C1", first_revision=1, last_revision=1): + def __call__(self, documents, digests) -> Policy: + catalog_data = documents[PolicyFile.CATALOG] + catalog = CatalogDocument.model_validate(catalog_data) + enrollment = catalog.enrollment + catalogs = catalog.catalogs.model_dump(mode="json") + if len(enrollment.models) != len(set(enrollment.models)): + raise ValueError("Duplicate enrolled model") + if not set(enrollment.models) <= catalog.catalogs.models.keys(): + raise ValueError("Enrollment references an unknown model") + if set(enrollment.check_groups) != { + "submission", + "system", + "curve", + "point", + "diagnostics", + }: + raise ValueError("Enrollment must include each policy check group") + if len(enrollment.check_groups) != len(set(enrollment.check_groups)): + raise ValueError("Duplicate enrolled check group") + groups: list[tuple[PolicyFile, Diagnostics | GroupDocument]] = [ + (PolicyFile.CATALOG, catalog.diagnostics) + ] + for file in PolicyFile: + if file is PolicyFile.CATALOG: + continue + parsed_group = GroupDocument.model_validate(documents[file]) + if parsed_group.scope is not file.scope: + raise ValueError(f"Wrong scope in {file.value}") + groups.append((file, parsed_group)) + rules: dict[str, Rule] = {} + for file, group in groups: + for id_, definition in group.checks.items(): + if id_ in rules: + raise ValueError(f"Duplicate rule ID {id_}") + metadata = Metadata.model_validate( + { + "scope": group.scope, + "severity": group.severity, + **{ + key: value + for key, value in definition.items() + if key in Metadata.model_fields + }, + } + ) + parameters = { + key: value + for key, value in definition.items() + if key not in Metadata.model_fields + } + try: + validate_requirements(metadata.kind, parameters) + except ValueError as error: + raise ValueError(f"Rule {id_}: {error}") from error + validate_catalog_references( + parameters, catalog_data["catalogs"], catalog_data["enrollment"] + ) + references = evidence_references(parameters) + for condition in (metadata.applies_to, metadata.unless): + if ( + condition + and condition.models + and not set(condition.models) <= set(enrollment.models) + ): + raise ValueError( + f"Rule {id_} condition references an unenrolled model" + ) + rules[id_] = Rule( + id_, + metadata.kind, + metadata.scope, + metadata.severity, + metadata.applies_to, + metadata.unless, + frozenset(metadata.requires), + freeze(parameters), + references, + file, + needs_model(parameters), + ) + overrides = [] + seen_overrides = set() + for override in enrollment.overrides: + if override.id in seen_overrides or override.rule not in rules: + raise ValueError("Duplicate override ID or unknown override rule") + seen_overrides.add(override.id) + rule = rules[override.rule] + try: + validate_requirements( + rule.kind, {**rule.requirements, **override.requirements} + ) + except ValueError as error: + raise ValueError( + f"Override {override.id} for {rule.id}: {error}" + ) from error + validate_catalog_references( + override.requirements, + catalogs, + catalog_data["enrollment"], + ) + evidence_references(override.requirements) + if override.applies_to.models and not set( + override.applies_to.models + ) <= set(enrollment.models): + raise ValueError("Override references an unenrolled model") + overrides.append( + Override( + override.id, + override.rule, + override.reason, + override.applies_to, + override.enabled, + freeze(override.requirements), + ) + ) + digest = hashlib.sha256( + "".join(f"{file.value}:{digests[file]}\n" for file in PolicyFile).encode() + ).hexdigest() + return Policy( + Release(version=catalog.version, revision=catalog.revision), + freeze(catalogs), + frozenset(enrollment.models), + tuple(rules.values()), + tuple(overrides), + freeze(documents), + MappingProxyType(dict(digests)), + digest, + ) diff --git a/src/inference_endpoint/validation/schemas/values_v1.py b/src/inference_endpoint/validation/schemas/values_v1.py new file mode 100644 index 000000000..dc4e4b8d2 --- /dev/null +++ b/src/inference_endpoint/validation/schemas/values_v1.py @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Catalog shapes for cohort policy revision 1.""" + +import math +from typing import Literal + +from pydantic import Field, JsonValue, model_validator + +from inference_endpoint.config.schema import LoadPatternType + +from ..types import ( + AccuracyKind, + CohortId, + CommitId, + Cooling, + DecodeMethod, + Division, + FrozenModel, + Identifier, + OfflineMode, + Release, + SampleUnit, + Scope, + Severity, +) + + +class Dataset(FrozenModel): + sample_count: int | None = Field(strict=True, gt=0) + sample_unit: SampleUnit = SampleUnit.SAMPLE + is_legacy: bool = Field(default=False, strict=True) + + +class Checkpoint(FrozenModel): + repository: Identifier + revision: CommitId + + +class DecodeHead(Checkpoint): + model: Identifier + method: DecodeMethod + bundled: bool = Field(default=False, strict=True) + approved_cohort: CohortId | None = None + + +class Accuracy(FrozenModel): + kind: AccuracyKind + datasets: tuple[Identifier, ...] = Field(min_length=1) + metrics: dict[Identifier, dict[str, JsonValue]] | None = None + inline_minimum_percent: float | None = Field(default=None, ge=0, le=100) + swebench_mean_minimum_percent: float | None = Field(default=None, ge=0, le=100) + full_run_osl_range: tuple[int, int] | None = None + + @model_validator(mode="after") + def check_profile(self): + if len(set(self.datasets)) != len(self.datasets): + raise ValueError("Duplicate accuracy dataset") + if self.kind is AccuracyKind.SINGLE_TURN: + if not self.metrics or any( + value is not None + for value in ( + self.inline_minimum_percent, + self.swebench_mean_minimum_percent, + self.full_run_osl_range, + ) + ): + raise ValueError( + "Single-turn accuracy requires only its metric thresholds" + ) + elif self.metrics is not None or any( + value is None + for value in ( + self.inline_minimum_percent, + self.swebench_mean_minimum_percent, + self.full_run_osl_range, + ) + ): + raise ValueError( + "Agentic accuracy requires inline, SWE-bench and OSL thresholds" + ) + if self.full_run_osl_range is not None: + lower, upper = self.full_run_osl_range + if lower <= 0 or upper < lower: + raise ValueError("Invalid full-run OSL range") + return self + + +class Model(FrozenModel): + accuracy: Accuracy + + +class Catalogs(FrozenModel): + load_pattern: tuple[LoadPatternType, ...] + divisions: tuple[Division, ...] + offline_modes: tuple[OfflineMode, ...] + region_names: tuple[Identifier, ...] + declared_regions: tuple[Identifier, ...] + mandatory_accuracy_bands: tuple[Identifier, ...] + datasets: dict[Identifier, Dataset] + models: dict[Identifier, Model] + approved_sped_decode_heads: tuple[DecodeHead, ...] + approved_client_revisions: tuple[CommitId, ...] + approved_checkpoints: dict[Identifier, tuple[Checkpoint, ...]] + submission_flags: dict[str, JsonValue] + disclosure: dict[str, JsonValue] + steady_state: dict[str, JsonValue] + regions: dict[str, JsonValue] + power: dict[str, JsonValue] + metrics: dict[str, JsonValue] + + @model_validator(mode="after") + def check_references(self): + overhead = self.power.get("cooling_overhead") + if not isinstance(overhead, dict) or set(overhead) != set(Cooling): + raise ValueError( + "Power cooling_overhead must define liquid, air, and mixed" + ) + if any( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + for value in overhead.values() + ): + raise ValueError( + "Cooling overhead fractions must be nonnegative numeric values" + ) + for name, model in self.models.items(): + if not set(model.accuracy.datasets) <= self.datasets.keys(): + raise ValueError(f"Unknown accuracy dataset for {name}") + if not self.approved_checkpoints.keys() <= self.models.keys(): + raise ValueError("Checkpoint catalog references unknown models") + for head in self.approved_sped_decode_heads: + if head.model not in self.models: + raise ValueError("Decode head references unknown model") + return self + + +class GroupDocument(Release): + scope: Scope + severity: Severity = Severity.ERROR + checks: dict[Identifier, dict[str, JsonValue]] = Field(min_length=1) + + +class Diagnostics(FrozenModel): + scope: Literal[Scope.VALIDATOR] + severity: Severity = Severity.WARNING + checks: dict[Identifier, dict[str, JsonValue]] diff --git a/src/inference_endpoint/validation/types.py b/src/inference_endpoint/validation/types.py new file mode 100644 index 000000000..e700d423e --- /dev/null +++ b/src/inference_endpoint/validation/types.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Closed vocabulary and policy identities shared by parsing and planning.""" + +from enum import StrEnum +from typing import Annotated + +from pydantic import AfterValidator, BaseModel, ConfigDict, Field, StringConstraints + + +def _canonical_identifier(value: str) -> str: + if value != value.strip(): + raise ValueError("Identifiers must not contain surrounding whitespace") + return value + + +Identifier = Annotated[ + str, StringConstraints(min_length=1), AfterValidator(_canonical_identifier) +] +CohortId = Annotated[str, StringConstraints(pattern=r"^\d{4}-(0[1-9]|1[0-2])-C[01]$")] +CommitId = Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{40}$")] + + +class FrozenModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +class Release(FrozenModel): + version: CohortId + revision: int = Field(strict=True, ge=1) + + +class Scope(StrEnum): + SUBMISSION = "submission" + SYSTEM = "system" + CURVE = "curve" + POINT = "point" + VALIDATOR = "validator" + IMPLEMENTATION = "implementation" + + +class Severity(StrEnum): + ERROR = "error" + WARNING = "warning" + INFO = "info" + + +class PolicyFile(StrEnum): + CATALOG = "catalog.yaml" + SUBMISSION = "submission_checks.yaml" + SYSTEM = "system_checks.yaml" + CURVE = "curve_checks.yaml" + POINT = "point_checks.yaml" + + @property + def scope(self) -> Scope: + return { + self.CATALOG: Scope.VALIDATOR, + self.SUBMISSION: Scope.SUBMISSION, + self.SYSTEM: Scope.SYSTEM, + self.CURVE: Scope.CURVE, + self.POINT: Scope.POINT, + }[self] + + +class OfflineMode(StrEnum): + NONE = "none" + DEDICATED = "dedicated" + ELECTED = "elected" + + +class Division(StrEnum): + STANDARDIZED = "Standardized" + SERVICED = "Serviced" + RDI = "RDI" + + +class Declaration(StrEnum): + WARMUP = "warmup" + STEADY_STATE = "steady_state" + POWER_DESCRIPTOR = "power_descriptor" + + +class EvidenceKey(StrEnum): + COMPUTED_REGIONS = "computed_regions" + WARMUP = "warmup" + STEADY_STATE = "steady_state" + SEED_CATALOG = "seed_catalog" + APPROVED_SPED_DECODE_HEADS = "approved_sped_decode_heads" + POINT_CONFIG = "point_config" + POWER_COMPUTATION = "power_computation" + POINT_POWER = "point_power" + + +class Decision(StrEnum): + READY = "ready" + EXCLUDED = "excluded" + BLOCKED = "blocked" + + +class Match(StrEnum): + YES = "yes" + NO = "no" + UNKNOWN = "unknown" + + +class AccuracyKind(StrEnum): + SINGLE_TURN = "single_turn" + AGENTIC = "agentic" + + +class SampleUnit(StrEnum): + SAMPLE = "sample" + TRAJECTORY = "trajectory" + INSTANCE = "instance" + + +class DecodeMethod(StrEnum): + DSPARK = "dspark" + MTP = "mtp" + + +class Cooling(StrEnum): + LIQUID = "liquid" + AIR = "air" + MIXED = "mixed" diff --git a/src/inference_endpoint/validation/vocabulary.py b/src/inference_endpoint/validation/vocabulary.py new file mode 100644 index 000000000..739e61589 --- /dev/null +++ b/src/inference_endpoint/validation/vocabulary.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Evaluator kinds and explicit evidence addresses admitted by policy parsers.""" + +from enum import StrEnum + + +class CheckKind(StrEnum): + ACCURACY_COVERAGE = "accuracy_coverage" + ACCURACY_GATE = "accuracy_gate" + ACCURACY_PRESENCE = "accuracy_presence" + ARTIFACT_BINDING = "artifact_binding" + ARTIFACT_SCHEMA = "artifact_schema" + CATALOG_INTEGRITY = "catalog_integrity" + COHORT_IDENTIFIER = "cohort_identifier" + COLLECTION_SIZE = "collection_size" + COMPARISON = "comparison" + CONSISTENCY = "consistency" + COVERAGE = "coverage" + DATASET_COUNT = "dataset_count" + DATASET_MINIMUM = "dataset_minimum" + DERIVED_METRIC = "derived_metric" + DISCLOSURE = "disclosure" + DRAFTER_BINDING = "drafter_binding" + DURATION = "duration" + FIELD_CONSTRAINTS = "field_constraints" + ISSUANCE = "issuance" + MEMBERSHIP = "membership" + NUMERIC_VALIDITY = "numeric_validity" + OFFLINE = "offline" + PATH_RESOLUTION = "path_resolution" + POWER = "power" + PRESENCE = "presence" + REGION_BASIS = "region_basis" + REGION_BOUNDARIES = "region_boundaries" + REGION_PLACEMENT = "region_placement" + REPORT = "report" + SEED_BINDING = "seed_binding" + STEADY_STATE = "steady_state" + WARMUP = "warmup" + + +class EvidenceReference(StrEnum): + ACCURACY_SWE_BENCH_EVALUATED_INSTANCE_COUNT = ( + "accuracy.swe_bench.evaluated_instance_count" + ) + ACCURACY_SWE_BENCH_EXTRAS_NUM_INSTANCES = "accuracy.swe_bench.extras.num_instances" + ACCURACY_SWE_BENCH_EXTRAS_SWEBENCH_TEMPLATE = ( + "accuracy.swe_bench.extras.swebench_template" + ) + CURVE_FIRST_DECLARED_MODEL_NAME = "curve.first_declared_model_name" + CURVE_MAX_SUPPORTED_CONCURRENCY = "curve.max_supported_concurrency" + CURVE_MODEL_DIRECTORY_NAME = "curve.model_directory_name" + CURVE_PARSED_POINTS = "curve.parsed_points" + CURVE_POINT_DIRECTORIES = "curve.point_directories" + IMPLEMENTATION_README = "implementation.readme" + MODEL_ACCURACY_DATASETS = "model.accuracy.datasets" + MODEL_ACCURACY_FULL_RUN_OSL_RANGE = "model.accuracy.full_run_osl_range" + MODEL_ACCURACY_INLINE_MINIMUM_PERCENT = "model.accuracy.inline_minimum_percent" + MODEL_ACCURACY_SWEBENCH_MEAN_MINIMUM_PERCENT = ( + "model.accuracy.swebench_mean_minimum_percent" + ) + POINT_CHECKPOINT_REPOSITORY = "point.checkpoint.repository" + POINT_CHECKPOINT_REVISION = "point.checkpoint.revision" + POINT_CONCURRENCY = "point.concurrency" + POINT_DATASET = "point.dataset" + POINT_DIRECTORY = "point.directory" + POINT_DIRECTORY_CONCURRENCY = "point.directory_concurrency" + POINT_DP_SHORTFALL = "point.dp_shortfall" + POINT_MODEL_NAME = "point.model_name" + POINT_NODES_USED = "point.nodes_used" + POINT_OFFLINE = "point.offline" + POINT_POWER_KW = "point.power_kw" + POINT_REGION = "point.region" + POINT_RESULT_SUMMARY_JSON = "point.result_summary.json" + POINT_RUNTIME_SETTINGS_AGENTIC_INFERENCE_NUM_TRAJECTORIES_TO_ISSUE = ( + "point.runtime_settings.agentic_inference.num_trajectories_to_issue" + ) + POINT_RUNTIME_SETTINGS_LOAD_PATTERN = "point.runtime_settings.load_pattern" + POINT_RUNTIME_SETTINGS_STREAM_ALL_CHUNKS = ( + "point.runtime_settings.stream_all_chunks" + ) + POINT_RUNTIME_SETTINGS_WARMUP_SALT = "point.runtime_settings.warmup.salt" + POINT_SEED_SET = "point.seed_set" + POINT_SHARED_DOCS = "point.shared_docs" + POINT_SHARED_SRC = "point.shared_src" + POINT_SYSTEM_DESC_JSON = "point.system_desc.json" + POINT_SYSTEM_DESCRIPTION = "point.system_description" + POINT_SYSTEM_DESCRIPTION_SYSTEM_NAME = "point.system_description.system_name" + POINT_SYSTEM_DESCRIPTION_TPS_UTILIZATION = ( + "point.system_description.tps_utilization" + ) + POINT_TARGET_COHORT = "point.target_cohort" + POINT_WARMUP = "point.warmup" + POINT_YAML = "point.yaml" + RESULT_SUMMARY_DURATION_NS = "result_summary.duration_ns" + RESULT_SUMMARY_E2E_AVG_INTERACTIVITY = "result_summary.e2e_avg_interactivity" + RESULT_SUMMARY_GIT_SHA = "result_summary.git_sha" + RESULT_SUMMARY_JSON = "result_summary.json" + RESULT_SUMMARY_N_SAMPLES_COMPLETED = "result_summary.n_samples_completed" + RESULT_SUMMARY_N_SAMPLES_FAILED = "result_summary.n_samples_failed" + RESULT_SUMMARY_N_SAMPLES_ISSUED = "result_summary.n_samples_issued" + RESULT_SUMMARY_OUTPUT_SEQUENCE_LENGTHS_TOTAL = ( + "result_summary.output_sequence_lengths.total" + ) + RESULT_SUMMARY_OUTPUT_SEQUENCE_LENGTHS_FULL_RUN_OUTPUT_SEQUENCE_LENGTHS_AVG = ( + "result_summary.output_sequence_lengths_full_run.output_sequence_lengths.avg" + ) + RESULT_SUMMARY_RAW_DERIVED_SYSTEM_TPS = "result_summary.raw_derived_system_tps" + RESULT_SUMMARY_SYSTEM_TPS = "result_summary.system_tps" + RESULT_SUMMARY_SYSTEM_TPS_PER_KW = "result_summary.system_tps_per_kw" + RESULT_SUMMARY_TPOT_PERCENTILES_90 = "result_summary.tpot.percentiles.90" + RESULT_SUMMARY_TPS_PER_USER = "result_summary.tps_per_user" + SUBMISSION_IMPLEMENTATION_DIRECTORIES = "submission.implementation_directories" + SUBMISSION_ROOT = "submission.root" + SUBMISSION_SYSTEM_DIRECTORIES = "submission.system_directories" + SYSTEM_MODEL_DIRECTORIES = "system.model_directories" diff --git a/tests/fixtures/validation/policy-fingerprints.json b/tests/fixtures/validation/policy-fingerprints.json new file mode 100644 index 000000000..33650e087 --- /dev/null +++ b/tests/fixtures/validation/policy-fingerprints.json @@ -0,0 +1,97 @@ +{ + "version": "2026-10-C1", + "revision": 1, + "catalog_digest": "b6d1e4363c4cfd80a255951f3f08fd8a21913918fb586ced16eff6b51e7beee6", + "enrollment_digest": "4d4f3a4087945435ffacee743906a9597be630d8117aca1a26bf0a4bd3a11cab", + "rules": { + "submission/path-exists": "10d1f797fff02890ddd7e3473732b96c44520d23952f9b7493fc576f6e5bf6a3", + "submission/required-dir": "17674a08b2897de6897d22270998e999d0ff8ac118a35d263555604c727a3f9d", + "submission/system-results-dir": "72456038d0d00ed12a7e3d58eda518b86b69aea7b019fad32b13a8310359704c", + "submission/src-dir": "a5b4a62887d8636f2f758a717100b68bfe31f3107648536236e990569f971cfa", + "submission/src-readme": "5754b3df0d5389985d5a476d630274d9fa1143e6cfa04be8616e119a55b5ac3b", + "submission/accuracy-present": "2f0dc1d59739717847bcb1519c78d6f2bbd871e9a1e666e6daa99b31f0a0824c", + "submission/system-name-consistency": "7ef9d2b5a293776dd30c1fea663e1b401678c75f826d89743cf0706a628784e6", + "system/benchmark-model-dir": "517f85dedefe201e639a74d83c581cd9efa481731848761d9ade76df7326bb3e", + "system/power-descriptor": "7b3b36310067d0772fb58c927e219d68c73881c25f62637fcb98498b6e9639a7", + "system/power-estimated": "d561adae981297c84fe746712f3e4008857e2e9c947c9d69f41e6595c3409fd3", + "system/system-description-across-curves": "6ff9a22fb2d7bc71fcdb91c7a13c8bee6bec0f8cfaefc5c5f1ad80c901c23f43", + "curve/point-dirs": "7f5c829ce0379813c1a490dcda57449675767ba7b0bd6849ab91d8ad3c2233ef", + "curve/measurement-points-present": "dafd79462c87ff519281ad09ae256ddafee22f2c681c46c60589dabfa514c464", + "curve/system-description-consistency": "77e9c8d89f667c8f830edd8ddf77191d177cab78a4ad2291e14e185b72c29782", + "curve/max-concurrency-declared": "54b6c7e9800b8bfe5099da91ffaecb25e462fccecf5098a89a6cfef10aeeb0eb", + "curve/region-basis": "2b2431e4f01f66b9a77fc7ecab0e8e90e8e76a7f83663eeb235dbdb08116ba54", + "curve/region-computation": "24c56d29544e482c4f9931f399ade3d232fc2744c77c16174d258dd74a091a21", + "curve/model-name-valid": "b3a33471bb36c3bb0d57eb3b8d015f95e2176aa530c85ce547a72dd0b560f788", + "curve/model-name-consistency": "8c6cac2996bacf763af57c2e654846dc52cea95345d495900fc7f35605a2147f", + "curve/config-consistency-model": "709507fdd16f1788b19ae7f6d7a3893417d8bc73f4acb24dcdafc95a843d1296", + "curve/config-consistency-dataset": "97539f45fbb7fdf9916bb16ec24607e5a194092186e2934601fc4d922a9ea5e2", + "curve/benchmark-type-consistency": "b2e5d794ae9e39c619b436cf01318b78651080a4b9f9bd993db536c3878f6c10", + "curve/point-count": "2314996964b653a78641df73cfac4c341f6c1260107e435eff0d982f4ea9ff15", + "curve/point-cap": "ef4565bdbce9fe5d548d4c758bd06acfdaf8614e032018f0cb22c2bd7b122cea", + "curve/ultra-low-concurrency-coverage": "a83bba002e1be6cdb4597b2e27597e97ed79abd1157ca9c1d7d285de4516486d", + "curve/low-concurrency-coverage": "c23a35ad020b892b5e769675433a4f3bd815a454d7a54800970675e81262e597", + "curve/med-concurrency-coverage": "1fa4671c42fe8a7b92f036e220419b7e85874cd132b76414146907e03c0e8166", + "curve/high-concurrency-coverage": "ad1180bb8571047a8c51aa297cfee8ff9f0ced96b1a732b3fabec97b1bfc4e14", + "curve/offline-point-present": "a8d496b614e4010ddfffe2799ff3585af83db6a28f21c92eadd47443360d66bf", + "curve/offline-ordering": "6580ae22479293be5d268977c0aec7eb6b0b09b93b7940cda1092af7d3cc246c", + "curve/tps-utilization": "000e3e3938029ed129495ad661900103d8b7daf49aaf9a463cef0333025b8e1b", + "curve/seed-set-consistency": "8d448c73d6012d37571a4786e3f2661669e7a0e4b787ec213d90bb8805fef215", + "curve/accuracy-coverage": "eb5722a35f0682b5da082689e02b1828fba8124259b10ef83e71a56ae45559e1", + "curve/accuracy-gate": "eaea400ff65e395339cf207162039a23f49c0a7abc8ee44c9b9714fc0cd8a98f", + "curve/accuracy-sample-count": "e36c67cb755ee8f69eef331574785d4e23c1faaa324ea6f01fa3226d6c5d8f22", + "curve/agentic-accuracy": "64e58a91f8eb5aad807abe29aaa7a719c79fcbb3cc369c8be6d966c8219cfc61", + "curve/agentic-accuracy-inline": "cc92cf9b4be30ce05c95d46da6bc1cb07d6098272224b554d05daf1975b8c09f", + "curve/agentic-accuracy-swebench": "af5ba468e3c4af612c1a0fb21006b82fdb68b2e648045df2d56c41734da0d79f", + "curve/agentic-osl-range": "e20feb64472b8faddcf31e08f3c18836c419ff239d942026bf275b0147597074", + "point/submission-flags": "6fd56145219a078dd947d79bd1a5587e9c45da258085ea0e0145d3d75e7330a9", + "point/swebench-template": "664ef45ae9124f181b8b1e947e25f29b6567ebb09c4ea192052c50678435dc15", + "point/agentic-trajectory-count": "5f22a6eea8421543f1a81e845f17be5e9f0ebbdd795d281e7889bc454430c18c", + "point/swebench-instance-count": "e9e031fe13262a9c7b60cf526def692e7d21085d84d0b10d5a9e2f575f3f42ea", + "point/endpoints-client-sha": "23b06e7e9a8dd4b39cf1cbd6809ec3f5bc2c80a49f460011b62f6dcb70ed8536", + "point/approved-checkpoint": "fd35d43f5c4c9f4bb268a80c6d365226dfc53a25902fa98bf378add73e4766c3", + "point/point-power": "6e75d380e17c6198e37ade0ac1ebd4598fc57c367a7994ff9543a75cb2106961", + "point/point-config-valid": "4bb6c62e68b0748f1a1b47110f0083029d0024958d17f4b8db38af21aed9dfaf", + "point/point-dirname-concurrency": "2296b404dacb1a6f15106be0dc2a6acea4d519d54e44c486e35c6ac78ec066fc", + "point/system-description-present": "856c0e732cba243a1bb153ab39176653a0d3a33fd33958095e564cd0e529f785", + "point/system-description-valid": "b53aee4801f00277cece609d210b3ce4496a257fe0d1d6a3ea2bda7b24b80c1d", + "point/result-summary-present": "9eaf56fa9c6a152f94563d2d8555a5435b887c7db9d3d9c5769c2f87c5b971d8", + "point/result-file-valid": "e5cebe2ee4158e9630ada483b0ac6596c3ee58608313a152ba8a460337f9f5f8", + "point/accuracy-valid": "3ac1f69a8b6103b319326669d255fae3c282888a188717dc2c9e5b73bdbdcbc6", + "point/point-disclosure-complete": "218ae026f7fdec937b0161af67519e60698a087c52eac59e579ae77203b254be", + "point/target-cohort": "6855eb240be6164648dccd441741230bca1783680c0b53a67cdf6b54e5466eba", + "point/shared-path-resolution": "4f6901ad243563cad240c70a7f0afed406639e5ee6d2f5ef4ea15b3ebc88ff14", + "point/offline-declared": "71281f294529c5665bde54020b1f8a5e204b9ed5735f9a135458eb2cae3c6a5a", + "point/region-declared": "e4340e52fed15f4a3dee74b73086f6688344057adddc45d737fb0bb88475719c", + "point/concurrency-in-range": "92b745ed6eba6d607bdaecae8f911838eda0940093f9422765f9fd3a374b44fe", + "point/region-placement": "283c684b2b3276f211af1a45f78f07702e804560a05a3e9993a05c58ad1c8283", + "point/load-pattern": "615a4d3f2faf27f07760c032cde07715ac17af76c9b5c1728635b45b8eeacace", + "point/streaming-config": "6b7d56890aed2d66ff24c2c1ee8a9b1b8cd4d465f94e8dc800b76ea9632f2fd9", + "point/warmup-present": "ade8ebb85859a4824b300ce75598921791c5b7f9cd9a9569d4729f27404aced1", + "point/warmup-logs-retained": "d63e4b1891824a0834adca73b95a192bbf3685536bbb4161f21d1a4cc72e7de4", + "point/warmup-salt": "091a57e9035d9a7f98e64cec9429bb5ed5b5a5a1b1f3b4f8510e191b7bc111f8", + "point/steady-state-valid": "39699685e3a49ff6db27ee6c02405878ef2362262f03f37088ac2a4a42ce5903", + "point/steady-state-consistency": "9f751df4221317b44d13882c357c21b87e1ddbd1d62a88e3ee9e3dd89d40a5ae", + "point/steady-state-basis": "c06a9eae5142326f1daab8db85b8a2bc8908f9d261ce3446d18b192b277f8177", + "point/point-duration": "d5d109fd65f15b494af5554f82214e4d37adbe32b9407dc91a29ebba5037eafa", + "point/min-query-count": "b1200c70c4f53a8a1411f98babcb75441c9288eee32cea915f853157207e0de8", + "point/metric-consistency-duration": "e8dd3ddb3fd5d339147e4218ed178b20c620f0a8bcc5997ce70babdddf7b556b", + "point/metric-consistency-accounting": "00600517efa73cb889b43fb4c31f7ae7b28de1bb8020da0a9c18e48369b08a03", + "point/metric-consistency-output-tokens": "a9464734c55d01306cff0baf8b90aecce9c1052379726f11354cf1cf741461af", + "point/metric-consistency-system-tps": "fb2fa8305218403a0bd99cca615f58f802d2dea78b646cc485fcb7b9bc20b1a8", + "point/metric-consistency-tpot-p90": "03379bcdce014772308db16a0fa6d3be734d33a742de4c28b8919a7f6debe6ba", + "point/metric-consistency-tps-per-user": "3d33681d535fcfb1173e80ddf4da57b2dfb401c39821f07851a873bb43fa17f0", + "point/metric-consistency-tps-per-kw": "a8423347c77fbc0cb9ff347c687f80307ffab6c9ffd07fb2c81203125c6b93db", + "point/agentic-metric-consistency": "2c08e062819cfb8f397938cc0c96030dcb34fb379e0681a0e79d211e6c768731", + "point/seed-config-legacy": "71b0b9e90407e7d0a8283abe23584727a267d19baa078f892c21802fb87160c1", + "point/seed-set-membership": "8d28659d6b0be81fe43d400a75aedf9bad54f68fd7ee8985ffe98055e536c338", + "point/seed-runtime-match": "530f81fec40bc8738b7b4f28268de0aad7a74e37c61d03e0391472b2cbae6796", + "point/seed-set-adoption": "b4f9895bcb58c00c69add9cdb33058c251ec3a1d208a10c5fc01693ac33684a5", + "point/approved-drafter": "eafdddb450939724ddfe0f45026caf2b18ada8f6e599e578a342e19ab3836b0c", + "point/drafter-approval-lead-time": "d4d43369592c41a488033384eacc4cc7c3cdfac4c715d981959439d6f611e066", + "point/nodes-used": "3b0164f362ed651e2feb2ed00d2ae790c974ac74a3e364008699f7a1166be2e0", + "point/maximal-engagement": "7ed77d2c82deb62d3d8396eabe191202a61180fa43d9625ad925bcb64da554db", + "point/point-rules-skipped": "d469f8eff6daf0307fc7309f0b2d4db6b05f0649f649ec56a5017efaa67d0021", + "diagnostics/seed-set-registry": "7670c90fb34c6459fc0a1808c999337db09d7e7d9a95283521d9235cddd4d647", + "diagnostics/drafter-list-registry": "5b753c1c9d92828a13bbe866841833612e283698771a6c6992b7c717532c6904" + } +} diff --git a/tests/unit/validation/test_cohorts.py b/tests/unit/validation/test_cohorts.py new file mode 100644 index 000000000..11d54eb69 --- /dev/null +++ b/tests/unit/validation/test_cohorts.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Tests for publication-cohort identifiers and arithmetic (§4.2, §4.6).""" + +from __future__ import annotations + +import pytest + +from inference_endpoint.validation.cohorts import Cohort, adoption_window + +pytestmark = pytest.mark.unit + + +class TestParse: + @pytest.mark.parametrize( + "text, expected", + [ + ("2026-10-C1", Cohort(2026, 10, 1)), + ("2026-01-C0", Cohort(2026, 1, 0)), + (" 2026-10-C1 ", Cohort(2026, 10, 1)), + ], + ) + def test_valid(self, text: str, expected: Cohort) -> None: + assert Cohort.parse(text) == expected + + @pytest.mark.parametrize( + "text", + [ + "", + "2026-10", + "2026-10-C2", # only two cohorts per month + "2026-13-C0", # month out of range + "2026-00-C0", + "26-10-C1", + "2026-10-c1", + "not-a-cohort", + ], + ) + def test_rejected(self, text: str) -> None: + assert Cohort.parse(text) is None + + def test_round_trips_through_str(self) -> None: + for text in ("2026-01-C0", "2026-12-C1", "2030-06-C0"): + assert str(Cohort.parse(text)) == text + + +class TestSuccession: + def test_first_cohort_of_a_month_yields_the_second(self) -> None: + assert str(Cohort(2026, 10, 0).next()) == "2026-10-C1" + + def test_second_cohort_rolls_into_the_next_month(self) -> None: + assert str(Cohort(2026, 10, 1).next()) == "2026-11-C0" + + def test_december_rolls_the_year(self) -> None: + assert str(Cohort(2026, 12, 1).next()) == "2027-01-C0" + + def test_succession_is_strictly_increasing(self) -> None: + """Ordering must agree with succession, or a window can't be compared.""" + c = Cohort(2026, 11, 0) + for _ in range(30): + assert c < c.next() + c = c.next() + + def test_two_cohorts_per_calendar_month(self) -> None: + """§4.2's bi-weekly cadence: a year advances in exactly 24 steps.""" + c = Cohort(2026, 1, 0) + for _ in range(24): + c = c.next() + assert str(c) == "2027-01-C0" + + +class TestAdoptionWindow: + def test_four_consecutive_cohorts_by_default(self) -> None: + """§4.6: the publication cohort and the following three.""" + window = adoption_window(Cohort(2026, 10, 1)) + assert [str(c) for c in window] == [ + "2026-10-C1", + "2026-11-C0", + "2026-11-C1", + "2026-12-C0", + ] + + def test_window_starts_at_the_publication_cohort(self) -> None: + published = Cohort(2027, 3, 0) + assert adoption_window(published)[0] == published + + def test_window_crosses_a_year_boundary(self) -> None: + window = adoption_window(Cohort(2026, 12, 0)) + assert [str(c) for c in window] == [ + "2026-12-C0", + "2026-12-C1", + "2027-01-C0", + "2027-01-C1", + ] + + @pytest.mark.parametrize("length", [1, 2, 4, 8]) + def test_length_is_configurable(self, length: int) -> None: + """A working-group change to the window is a call-site edit, not a rewrite.""" + assert len(adoption_window(Cohort(2026, 10, 1), length=length)) == length + + def test_window_is_contiguous(self) -> None: + window = adoption_window(Cohort(2026, 11, 1), length=6) + for earlier, later in zip(window, window[1:], strict=False): + assert earlier.next() == later + + def test_two_sets_published_two_cohorts_apart_overlap_by_two(self) -> None: + """§4.6: refresh every two cohorts, adoptable for four — so two sets overlap.""" + first = set(adoption_window(Cohort(2026, 10, 1))) + second = set(adoption_window(Cohort(2026, 11, 1))) + assert len(first & second) == 2 diff --git a/tests/unit/validation/test_coverage.py b/tests/unit/validation/test_coverage.py new file mode 100644 index 000000000..5a288e4e9 --- /dev/null +++ b/tests/unit/validation/test_coverage.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pinned source inventory and complete selection coverage for the policy draft.""" + +import pytest + +from inference_endpoint.validation import ( + Context, + Decision, + EvidenceKey, + bundled_policy_path, + load_policy, + plan_checks, +) +from inference_endpoint.validation.types import Declaration, Scope + +pytestmark = pytest.mark.unit + + +def test_every_rule_can_be_planned_for_a_supported_context(): + policy = load_policy(bundled_policy_path()) + subjects = [] + for scope in Scope: + for model, pattern in ( + ("llama3_1-8b", "concurrency"), + ("kimi-k3", "agentic_inference"), + ("kimi-k3", "max_throughput"), + ): + for offline in ("none", "dedicated", "elected"): + subjects.append( + Context.model_validate( + { + "id": f"{model}/{pattern}/{offline}", + "scope": scope, + "model_id": model, + "load_pattern": pattern, + "offline": offline, + "division": "Standardized", + "declared": list(Declaration), + "speculative_decoding": True, + "available": list(EvidenceKey), + "invalid": [], + } + ) + ) + subjects.append( + Context.model_validate( + { + "id": "bad-config", + "scope": "point", + "invalid": ["point_config"], + } + ) + ) + plan = plan_checks(policy, subjects) + ready = {check.rule.id for check in plan.ready} + assert ready == {rule.id for rule in policy.checks} + assert any(check.decision is Decision.EXCLUDED for check in plan.checks) + assert any(check.decision is Decision.BLOCKED for check in plan.checks) + + +def test_missing_evidence_blocks_every_selected_dependent_rule(): + policy = load_policy(bundled_policy_path()) + context = Context.model_validate( + { + "id": "agentic", + "scope": "point", + "model_id": "kimi-k3", + "load_pattern": "agentic_inference", + "offline": "none", + "declared": list(Declaration), + "speculative_decoding": True, + "invalid": [], + } + ) + for check in plan_checks(policy, [context]).checks: + if check.rule.requires and check.decision is not Decision.EXCLUDED: + assert check.decision is Decision.BLOCKED + assert check.missing == check.rule.requires diff --git a/tests/unit/validation/test_loader.py b/tests/unit/validation/test_loader.py new file mode 100644 index 000000000..dde40526d --- /dev/null +++ b/tests/unit/validation/test_loader.py @@ -0,0 +1,248 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Atomic policy loading, semantic fingerprints and bundle integrity.""" + +import hashlib +import json +import shutil +from pathlib import Path + +import pytest +import yaml + +from inference_endpoint.validation import bundled_policy_path, load_policy +from inference_endpoint.validation.schemas import BundleParser, parser_for +from inference_endpoint.validation.types import PolicyFile, Release, Scope +from inference_endpoint.validation.vocabulary import EvidenceReference + +pytestmark = pytest.mark.unit + +ROOT = Path(__file__).resolve().parents[3] +FINGERPRINTS = ROOT / "tests/fixtures/validation/policy-fingerprints.json" + + +def fingerprint(value): + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +@pytest.fixture +def policy_dir(tmp_path): + target = tmp_path / "2026-10-C1" + shutil.copytree(bundled_policy_path(), target) + return target + + +def modify(path, transform): + data = yaml.safe_load(path.read_text()) + transform(data) + path.write_text(yaml.safe_dump(data, sort_keys=False)) + + +def test_bundled_policy_matches_pinned_definitions(): + expected = json.loads(FINGERPRINTS.read_text()) + policy = load_policy(bundled_policy_path()) + monolith = policy.as_monolith() + assert policy.release.version == expected["version"] + assert policy.release.revision == expected["revision"] + assert fingerprint(monolith["catalogs"]) == expected["catalog_digest"] + assert fingerprint(monolith["enrollment"]) == expected["enrollment_digest"] + actual = { + f"{group}/{id_}": fingerprint(rule) + for group, definition in monolith["check_groups"].items() + for id_, rule in definition["checks"].items() + } + assert actual == expected["rules"] + assert len(actual) == 89 + + +def test_policy_is_immutable_and_preserves_source_hashes(policy_dir): + policy = load_policy(policy_dir) + with pytest.raises(TypeError): + policy.catalogs["datasets"]["aime25"]["sample_count"] = 1 + with pytest.raises(TypeError): + policy.checks[0].requirements["anything"] = 1 + for file, digest in policy.source_digests.items(): + assert ( + digest == hashlib.sha256((policy_dir / file.value).read_bytes()).hexdigest() + ) + (policy_dir / PolicyFile.POINT.value).write_text( + (policy_dir / PolicyFile.POINT.value).read_text() + "\n# Formatting-only edit\n" + ) + edited = load_policy(policy_dir) + assert edited.as_monolith() == policy.as_monolith() + assert edited.digest != policy.digest + + +@pytest.mark.parametrize("file", list(PolicyFile)) +def test_missing_any_file_rejects_whole_bundle(policy_dir, file): + (policy_dir / file.value).unlink() + with pytest.raises(ValueError, match="missing"): + load_policy(policy_dir) + + +@pytest.mark.parametrize( + "contents", + [ + "version: '2026-10-C1'\nversion: '2026-10-C1'\nrevision: 1\n", + "[]\n", + "version: [broken\n", + "version: '2026-10-C1'\nrevision: true\n", + ], +) +def test_malformed_or_duplicate_yaml_rejects_bundle(policy_dir, contents): + (policy_dir / PolicyFile.POINT.value).write_text(contents) + with pytest.raises(ValueError): + load_policy(policy_dir) + + +def test_extra_files_wrong_scope_and_mixed_revisions_are_rejected(policy_dir): + extra = policy_dir / "point.yaml" + extra.write_text("{}\n") + with pytest.raises(ValueError, match="unexpected"): + load_policy(policy_dir) + extra.unlink() + point = policy_dir / PolicyFile.POINT.value + modify(point, lambda data: data.update(revision=2)) + with pytest.raises(ValueError, match="same version/revision"): + load_policy(policy_dir) + modify(point, lambda data: data.update(revision=1, scope="system")) + with pytest.raises(ValueError, match="Wrong scope"): + load_policy(policy_dir) + + +@pytest.mark.parametrize( + "change", + [ + {"kind": "magic_function"}, + {"applies_to": {"load_pattern": ["COP"]}}, + {"applies_to": {"curve_type": "agentic"}}, + {"requires": ["region_ready"]}, + {"source": "point.runtime_settings.typo"}, + {"selector": "agentic"}, + {"catalog": "catalogs.missing"}, + ], +) +def test_unknown_dispatch_and_reference_values_are_rejected(policy_dir, change): + point = policy_dir / PolicyFile.POINT.value + modify( + point, lambda data: data["checks"]["agentic-trajectory-count"].update(change) + ) + with pytest.raises(ValueError): + load_policy(policy_dir) + + +def test_duplicate_ids_and_broken_catalog_bindings_are_rejected(policy_dir): + point = policy_dir / PolicyFile.POINT.value + modify( + point, + lambda data: data["checks"].update( + {"path-exists": {"kind": "presence", "target": "point.directory"}} + ), + ) + with pytest.raises(ValueError, match="Duplicate rule ID"): + load_policy(policy_dir) + modify(point, lambda data: data["checks"].pop("path-exists")) + modify( + policy_dir / PolicyFile.CATALOG.value, + lambda data: data["catalogs"]["datasets"].pop("cnn_dailymail"), + ) + with pytest.raises(ValueError, match="Unknown accuracy dataset"): + load_policy(policy_dir) + + +def test_evidence_addresses_are_enum_backed(): + policy = load_policy(bundled_policy_path()) + rule = next(rule for rule in policy.checks if rule.id == "agentic-trajectory-count") + assert rule.scope is Scope.POINT + assert ( + EvidenceReference.POINT_RUNTIME_SETTINGS_AGENTIC_INFERENCE_NUM_TRAJECTORIES_TO_ISSUE + in rule.references + ) + + +def test_parser_dispatch_has_no_latest_revision_fallback(): + assert ( + type(parser_for(Release(version="2026-10-C1", revision=1))).__name__ + == "ParserV1" + ) + for version, revision in (("2026-10-C1", 2), ("2027-01-C0", 1)): + with pytest.raises(ValueError, match="Unsupported"): + parser_for(Release(version=version, revision=revision)) + with pytest.raises(ValueError, match="Overlapping"): + + class Conflicting( + BundleParser, version="2026-10-C1", first_revision=1, last_revision=2 + ): + def __call__(self, documents, digests): + raise AssertionError("Must not execute") + + +@pytest.mark.parametrize( + "section", ["checks", "datasets", "models", "approved_checkpoints"] +) +def test_noncanonical_identifiers_cannot_replace_existing_entries(policy_dir, section): + file = PolicyFile.POINT if section == "checks" else PolicyFile.CATALOG + path = policy_dir / file.value + + def duplicate(data): + entries = data["checks"] if section == "checks" else data["catalogs"][section] + key = next(iter(entries)) + entries[f" {key} "] = entries[key] + + modify(path, duplicate) + with pytest.raises(ValueError, match="whitespace"): + load_policy(policy_dir) + + +def test_optional_offline_and_region_declarations_are_checked_only_when_present(): + policy = load_policy(bundled_policy_path()) + rules = {rule.id: rule for rule in policy.checks} + assert rules["offline-declared"].requirements["required"] is False + assert rules["region-declared"].requirements["required"] is False + + +def test_decode_head_can_record_its_publication_cohort(policy_dir): + catalog = policy_dir / PolicyFile.CATALOG.value + modify( + catalog, + lambda data: data["catalogs"]["approved_sped_decode_heads"][0].update( + approved_cohort="2026-06-C0" + ), + ) + assert ( + load_policy(policy_dir).catalogs["approved_sped_decode_heads"][0][ + "approved_cohort" + ] + == "2026-06-C0" + ) + modify( + catalog, + lambda data: data["catalogs"]["approved_sped_decode_heads"][0].update( + approved_cohort="2026-06-C9" + ), + ) + with pytest.raises(ValueError): + load_policy(policy_dir) + + +@pytest.mark.parametrize("revision", ["abc123", "A" * 40, "g" * 40]) +def test_client_approval_list_rejects_malformed_sha(policy_dir, revision): + modify( + policy_dir / "catalog.yaml", + lambda data: data["catalogs"].update(approved_client_revisions=[revision]), + ) + with pytest.raises(ValueError): + load_policy(policy_dir) + + +def test_client_approval_list_loads_full_shas(policy_dir): + revisions = ["a" * 40, "b" * 40] + modify( + policy_dir / "catalog.yaml", + lambda data: data["catalogs"].update(approved_client_revisions=revisions), + ) + assert tuple( + load_policy(policy_dir).catalogs["approved_client_revisions"] + ) == tuple(revisions) diff --git a/tests/unit/validation/test_planner.py b/tests/unit/validation/test_planner.py new file mode 100644 index 000000000..6381b518f --- /dev/null +++ b/tests/unit/validation/test_planner.py @@ -0,0 +1,207 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Selection, exclusion, dependency and override behavior.""" + +import shutil + +import pytest +import yaml + +from inference_endpoint.validation import ( + Conditions, + Context, + Decision, + EvidenceKey, + bundled_policy_path, + load_policy, + plan_checks, +) +from inference_endpoint.validation.types import Declaration, Match, OfflineMode + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def policy(): + return load_policy(bundled_policy_path()) + + +def point(**changes): + return Context.model_validate( + { + "id": "r16", + "scope": "point", + "model_id": "kimi-k3", + "load_pattern": "agentic_inference", + "offline": "none", + "division": "Standardized", + "declared": [], + "speculative_decoding": False, + "available": list(EvidenceKey), + "invalid": [], + **changes, + } + ) + + +def decisions(policy, context): + return {check.rule.id: check for check in plan_checks(policy, [context]).checks} + + +def test_patterns_select_model_family_checks(policy): + agentic = decisions(policy, point()) + assert agentic["agentic-trajectory-count"].decision is Decision.READY + assert agentic["approved-drafter"].decision is Decision.EXCLUDED + single = decisions( + policy, point(model_id="llama3_1-8b", load_pattern="concurrency") + ) + assert single["agentic-trajectory-count"].decision is Decision.EXCLUDED + assert single["point-config-valid"].decision is Decision.READY + curve = Context.model_validate( + { + "id": "llama3_1-8b", + "scope": "curve", + "model_id": "llama3_1-8b", + "load_pattern": "concurrency", + "available": ["computed_regions"], + } + ) + assert decisions(policy, curve)["accuracy-gate"].decision is Decision.READY + assert ( + decisions(policy, curve)["agentic-accuracy-inline"].decision + is Decision.EXCLUDED + ) + + +def test_exclusion_precedes_missing_dependencies(policy): + checks = decisions( + policy, point(offline="dedicated", load_pattern="max_throughput", available=[]) + ) + assert checks["concurrency-in-range"].decision is Decision.EXCLUDED + assert checks["load-pattern"].decision is Decision.EXCLUDED + assert checks["agentic-trajectory-count"].decision is Decision.EXCLUDED + + +def test_unknown_classification_blocks_without_hiding_unconditional_checks(policy): + checks = decisions(policy, point(load_pattern=None, offline=None)) + assert checks["agentic-trajectory-count"].decision is Decision.BLOCKED + assert checks["concurrency-in-range"].decision is Decision.BLOCKED + assert checks["result-file-valid"].decision is Decision.READY + + +def test_invalid_config_does_not_hide_summary_schema_checks(policy): + checks = decisions( + policy, + point( + available=[ + key for key in EvidenceKey if key is not EvidenceKey.POINT_CONFIG + ], + invalid=[EvidenceKey.POINT_CONFIG], + ), + ) + assert checks["result-file-valid"].decision is Decision.READY + assert checks["point-rules-skipped"].decision is Decision.READY + assert checks["agentic-trajectory-count"].decision is Decision.BLOCKED + assert checks["agentic-trajectory-count"].missing == {EvidenceKey.POINT_CONFIG} + + +def test_missing_seed_catalog_blocks_instead_of_deselecting(policy): + checks = decisions(policy, point(available=[EvidenceKey.POINT_CONFIG])) + # Catalog dependency is independent of the artifact's declared seed values. + assert checks["seed-set-membership"].decision is Decision.BLOCKED + assert EvidenceKey.SEED_CATALOG in checks["seed-set-membership"].missing + + +def test_optional_warmup_and_power_are_selected_by_declared_facts(policy): + no_warmup = decisions(policy, point()) + assert no_warmup["warmup-logs-retained"].decision is Decision.EXCLUDED + warmup = decisions(policy, point(declared=[Declaration.WARMUP])) + assert warmup["warmup-logs-retained"].decision is Decision.READY + for declared, expected in ( + ([], Decision.EXCLUDED), + (["power_descriptor"], Decision.READY), + ): + context = Context.model_validate( + { + "id": "sys", + "scope": "system", + "division": "RDI", + "declared": declared, + } + ) + assert decisions(policy, context)["power-descriptor"].decision is expected + + +def test_curve_member_filter_does_not_exclude_the_whole_curve(policy): + included = point(id="r16") + offline = point(id="r4096", offline="dedicated", load_pattern="max_throughput") + curve = Context.model_validate( + { + "id": "kimi", + "scope": "curve", + "model_id": "kimi-k3", + "members": [included, offline], + } + ) + check = decisions(policy, curve)["benchmark-type-consistency"] + assert check.decision is Decision.READY + assert check.selected_members == ("r16",) + unknown = point(id="broken", offline=None) + curve = curve.model_copy(update={"members": (included, unknown)}) + assert ( + decisions(policy, curve)["benchmark-type-consistency"].decision + is Decision.BLOCKED + ) + + +def test_conditions_and_contexts_reject_contradictory_evidence(): + with pytest.raises(ValueError): + Conditions.model_validate({"declared": ["warmup"], "not_declared": ["warmup"]}) + with pytest.raises(ValueError): + point(invalid=["point_config"]) + condition = Conditions.model_validate( + {"offline": ["none", "elected"], "speculative_decoding": False} + ) + assert condition(point()) is Match.YES + assert condition(point(offline=OfflineMode.DEDICATED)) is Match.NO + assert condition(point(offline=None)) is Match.UNKNOWN + + +def test_model_override_changes_only_selected_context(tmp_path): + directory = tmp_path / "2026-10-C1" + shutil.copytree(bundled_policy_path(), directory) + path = directory / "catalog.yaml" + data = yaml.safe_load(path.read_text()) + override = { + "id": "kimi-count-exception", + "rule": "agentic-trajectory-count", + "reason": "Fixture exception", + "applies_to": {"models": ["kimi-k3"]}, + "requirements": {"operator": "equal"}, + } + data["enrollment"]["overrides"] = [override] + path.write_text(yaml.safe_dump(data, sort_keys=False)) + policy = load_policy(directory) + check = decisions(policy, point())["agentic-trajectory-count"] + assert check.override == "kimi-count-exception" + assert check.rule.requirements["operator"] == "equal" + other = decisions(policy, point(model_id="deepseek-v4_1-flash"))[ + "agentic-trajectory-count" + ] + assert other.override is None + assert other.rule.requirements["operator"] == "positive_multiple" + data["enrollment"]["overrides"].append({**override, "id": "overlapping"}) + path.write_text(yaml.safe_dump(data, sort_keys=False)) + with pytest.raises(ValueError, match="Overlapping overrides"): + plan_checks(load_policy(directory), [point()]) + + +def test_planning_keeps_independent_checks_and_rejects_unknown_models(policy): + checks = decisions(policy, point(model_id="unapproved", available=[])) + assert checks["approved-checkpoint"].decision is Decision.BLOCKED + assert checks["swebench-template"].decision is Decision.BLOCKED + assert checks["result-file-valid"].decision is Decision.READY + with pytest.raises(ValueError, match="Duplicate"): + plan_checks(policy, [point(), point()]) + with pytest.raises(ValueError, match="at least one"): + plan_checks(policy, []) diff --git a/tests/unit/validation/test_policy_contracts.py b/tests/unit/validation/test_policy_contracts.py new file mode 100644 index 000000000..3af47c9c9 --- /dev/null +++ b/tests/unit/validation/test_policy_contracts.py @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: Apache-2.0 +import shutil + +import pytest +import yaml + +from inference_endpoint.validation import bundled_policy_path, load_policy + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def policy_dir(tmp_path): + directory = tmp_path / "2026-10-C1" + shutil.copytree(bundled_policy_path(), directory) + return directory + + +@pytest.mark.parametrize("value", [[], "false", 0, None]) +@pytest.mark.parametrize("override", [False, True]) +def test_boolean_requirement_types_are_strict(policy_dir, value, override): + path = policy_dir / ("catalog.yaml" if override else "point_checks.yaml") + document = yaml.safe_load(path.read_text()) + if override: + document["enrollment"]["overrides"].append( + { + "id": "bad-positivity", + "rule": "metric-consistency-tpot-p90", + "reason": "test", + "applies_to": {"models": ["gpt-oss-120b"]}, + "requirements": {"strictly_positive": value}, + } + ) + else: + document["checks"]["metric-consistency-tpot-p90"]["strictly_positive"] = value + path.write_text(yaml.safe_dump(document, sort_keys=False)) + with pytest.raises(ValueError, match="strictly_positive"): + load_policy(policy_dir) + + +def test_missing_required_operand_is_rejected_when_loading(policy_dir): + path = policy_dir / "point_checks.yaml" + document = yaml.safe_load(path.read_text()) + del document["checks"]["metric-consistency-duration"]["right"] + path.write_text(yaml.safe_dump(document, sort_keys=False)) + with pytest.raises(ValueError, match="right"): + load_policy(policy_dir) + + +def test_valid_override_retains_false_and_numeric_values(policy_dir): + path = policy_dir / "catalog.yaml" + document = yaml.safe_load(path.read_text()) + document["enrollment"]["overrides"].extend( + [ + { + "id": "no-positivity", + "rule": "metric-consistency-tpot-p90", + "reason": "test", + "applies_to": {"models": ["gpt-oss-120b"]}, + "requirements": {"strictly_positive": False}, + }, + { + "id": "numeric-floor", + "rule": "offline-ordering", + "reason": "test", + "applies_to": {"models": ["gpt-oss-120b"]}, + "requirements": {"concurrency_floor": 128}, + }, + ] + ) + path.write_text(yaml.safe_dump(document, sort_keys=False)) + overrides = { + override.id: override for override in load_policy(policy_dir).overrides + } + assert overrides["no-positivity"].requirements["strictly_positive"] is False + assert overrides["numeric-floor"].requirements["concurrency_floor"] == 128 + + +@pytest.mark.parametrize( + ("rule", "changes"), + [ + ("metric-consistency-duration", {"left": 12}), + ("metric-consistency-duration", {"right": {"kind": "constant", "value": True}}), + ("metric-consistency-duration", {"operator": "unknown"}), + ("point-duration", {"basis_precedence": ["unknown"]}), + ("seed-runtime-match", {"fields": []}), + ("metric-consistency-tpot-p90", {"source": None}), + ], +) +def test_malformed_nested_and_operation_contracts_are_rejected( + policy_dir, rule, changes +): + path = policy_dir / "point_checks.yaml" + document = yaml.safe_load(path.read_text()) + document["checks"][rule].update(changes) + path.write_text(yaml.safe_dump(document, sort_keys=False)) + with pytest.raises(ValueError, match=rule): + load_policy(policy_dir) diff --git a/tests/unit/validation/test_seed_catalog.py b/tests/unit/validation/test_seed_catalog.py new file mode 100644 index 000000000..89068fb97 --- /dev/null +++ b/tests/unit/validation/test_seed_catalog.py @@ -0,0 +1,246 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the published seed-set registry (§4.6 Seed Rotation).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from inference_endpoint.validation.catalogs.seeds import ( + SeedSetError, + bundled_seed_sets_path, + load_seed_sets, + parse_seed_catalog, +) + +pytestmark = pytest.mark.unit + +#: Literal published seed values guard against registry transcription errors. +_PR117_SET_A = { + "scheduler_rng_seed": 10487924139932647040, + "sample_index_rng_seed": 586478644936801402, + "model_seed": 9315206023656308754, +} + + +def _write_registry(path: Path, entries: list[dict]) -> Path: + path.write_text(yaml.safe_dump({"seed_sets": entries})) + return path + + +class TestBundledRegistry: + def test_mirrors_pr117(self) -> None: + seed_set = load_seed_sets()["A"] + assert seed_set.seeds == _PR117_SET_A + + def test_seeds_are_not_the_v07_constant(self) -> None: + """Published seed fields carry their independent registry values.""" + assert all(value != 42 for value in load_seed_sets()["A"].seeds.values()) + + def test_adoption_window_is_derived_from_the_publication_cohort(self) -> None: + """§4.6: adoptable for the publication cohort and the three following it.""" + assert load_seed_sets()["A"].cohorts == ( + "2026-10-C1", + "2026-11-C0", + "2026-11-C1", + "2026-12-C0", + ) + + def test_bundled_file_ships_with_the_package(self) -> None: + assert bundled_seed_sets_path().is_file() + + +class TestRegistryShapes: + """Upstream nests under `cohort:`; the older flat form is still read.""" + + def test_upstream_cohort_shape(self, tmp_path: Path) -> None: + path = tmp_path / "seeds.yaml" + path.write_text( + yaml.safe_dump( + { + "cohort": { + "version": 1.0, + "cohort-id": "2027-03-C0", + "seed_sets": [ + { + "id": "Z", + "scheduler_rng_seed": 1, + "sample_index_rng_seed": 2, + "model_seed": 3, + } + ], + } + } + ) + ) + z = parse_seed_catalog(path)["Z"] + assert z.cohorts == ("2027-03-C0", "2027-03-C1", "2027-04-C0", "2027-04-C1") + + def test_flat_shape_still_parses_but_declares_no_window( + self, tmp_path: Path + ) -> None: + """A file predating the cohort wrapper leaves the adoption test unevaluable.""" + path = tmp_path / "seeds.yaml" + path.write_text( + yaml.safe_dump( + { + "seed_sets": [ + { + "id": "Z", + "scheduler_rng_seed": 1, + "sample_index_rng_seed": 2, + "model_seed": 3, + } + ] + } + ) + ) + assert parse_seed_catalog(path)["Z"].cohorts == () + + def test_per_set_cohorts_override_the_derived_window(self, tmp_path: Path) -> None: + path = tmp_path / "seeds.yaml" + path.write_text( + yaml.safe_dump( + { + "cohort": { + "cohort-id": "2026-10-C1", + "seed_sets": [ + { + "id": "Z", + "scheduler_rng_seed": 1, + "sample_index_rng_seed": 2, + "model_seed": 3, + "cohorts": ["2030-01-C0"], + } + ], + } + } + ) + ) + assert parse_seed_catalog(path)["Z"].cohorts == ("2030-01-C0",) + + +class TestCatalogParsings: + def test_explicit_path_wins(self, tmp_path: Path) -> None: + path = _write_registry( + tmp_path / "seeds.yaml", + [ + { + "id": "Z", + "scheduler_rng_seed": 1, + "sample_index_rng_seed": 2, + "model_seed": 3, + } + ], + ) + sets = parse_seed_catalog(path) + assert list(sets) == ["Z"] + assert sets["Z"].seeds == { + "scheduler_rng_seed": 1, + "sample_index_rng_seed": 2, + "model_seed": 3, + } + + def test_environment_cannot_replace_published_seeds(self, tmp_path, monkeypatch): + monkeypatch.setenv("MLPERF_ENDPOINTS_SEED_SETS", str(tmp_path / "missing.yaml")) + assert load_seed_sets() == parse_seed_catalog(bundled_seed_sets_path()) + + def test_cohorts_are_read_when_present(self, tmp_path: Path) -> None: + """Forward-compatibility: the loader already reads the key §4.6 needs.""" + path = _write_registry( + tmp_path / "seeds.yaml", + [ + { + "id": "A", + "scheduler_rng_seed": 1, + "sample_index_rng_seed": 2, + "model_seed": 3, + "cohorts": ["2026-09-C0", "2026-09-C1"], + } + ], + ) + assert parse_seed_catalog(path)["A"].cohorts == ("2026-09-C0", "2026-09-C1") + + +class TestMalformedRegistries: + def test_missing_file(self, tmp_path: Path) -> None: + with pytest.raises(SeedSetError, match="Cannot read"): + parse_seed_catalog(tmp_path / "absent.yaml") + + def test_invalid_yaml(self, tmp_path: Path) -> None: + path = tmp_path / "seeds.yaml" + path.write_text("seed_sets: [\n") + with pytest.raises(SeedSetError, match="Invalid YAML"): + parse_seed_catalog(path) + + def test_not_a_mapping(self, tmp_path: Path) -> None: + path = tmp_path / "seeds.yaml" + path.write_text("- just\n- a\n- list\n") + with pytest.raises(SeedSetError, match="must be a mapping"): + parse_seed_catalog(path) + + def test_malformed_cohort_id(self, tmp_path: Path) -> None: + path = tmp_path / "seeds.yaml" + path.write_text( + yaml.safe_dump( + { + "cohort": { + "cohort-id": "not-a-cohort", + "seed_sets": [ + { + "id": "A", + "scheduler_rng_seed": 1, + "sample_index_rng_seed": 2, + "model_seed": 3, + } + ], + } + } + ) + ) + with pytest.raises(SeedSetError, match="YYYY-MM-C0/C1"): + parse_seed_catalog(path) + + def test_cohort_without_seed_sets(self, tmp_path: Path) -> None: + path = tmp_path / "seeds.yaml" + path.write_text(yaml.safe_dump({"cohort": {"cohort-id": "2026-10-C1"}})) + with pytest.raises(SeedSetError, match="cohort.seed_sets must be a list"): + parse_seed_catalog(path) + + def test_empty_seed_set_list(self, tmp_path: Path) -> None: + with pytest.raises(SeedSetError, match="defines no seed sets"): + parse_seed_catalog(_write_registry(tmp_path / "seeds.yaml", [])) + + def test_entry_without_id(self, tmp_path: Path) -> None: + path = _write_registry( + tmp_path / "seeds.yaml", + [{"scheduler_rng_seed": 1, "sample_index_rng_seed": 2, "model_seed": 3}], + ) + with pytest.raises(SeedSetError, match="has no 'id'"): + parse_seed_catalog(path) + + def test_entry_missing_a_seed(self, tmp_path: Path) -> None: + path = _write_registry( + tmp_path / "seeds.yaml", + [{"id": "A", "scheduler_rng_seed": 1, "model_seed": 3}], + ) + with pytest.raises(SeedSetError, match="must define integer"): + parse_seed_catalog(path) + + def test_entry_with_a_non_integer_seed(self, tmp_path: Path) -> None: + path = _write_registry( + tmp_path / "seeds.yaml", + [ + { + "id": "A", + "scheduler_rng_seed": "not-a-number", + "sample_index_rng_seed": 2, + "model_seed": 3, + } + ], + ) + with pytest.raises(SeedSetError, match="must define integer"): + parse_seed_catalog(path) From 2744459acd2cf37b090747b00de8d358977d76f9 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 01:33:13 -0700 Subject: [PATCH 04/18] feat(validation): calculate policy-driven system power --- .../validation/power/__init__.py | 3 + .../validation/power/calculation.py | 452 ++++++++++++++++++ .../validation/power/models.py | 216 +++++++++ .../unit/validation/test_policy_contracts.py | 38 ++ 4 files changed, 709 insertions(+) create mode 100644 src/inference_endpoint/validation/power/__init__.py create mode 100644 src/inference_endpoint/validation/power/calculation.py create mode 100644 src/inference_endpoint/validation/power/models.py diff --git a/src/inference_endpoint/validation/power/__init__.py b/src/inference_endpoint/validation/power/__init__.py new file mode 100644 index 000000000..29dedbdd2 --- /dev/null +++ b/src/inference_endpoint/validation/power/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Power evidence and policy-driven arithmetic.""" diff --git a/src/inference_endpoint/validation/power/calculation.py b/src/inference_endpoint/validation/power/calculation.py new file mode 100644 index 000000000..a1d9744c9 --- /dev/null +++ b/src/inference_endpoint/validation/power/calculation.py @@ -0,0 +1,452 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Provisioned power arithmetic driven by cohort policy values.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +from .models import ( + Components, + NodeSet, + PowerComputation, + ScaleUpNetwork, + SetPower, + SourcedValue, + SystemPower, +) + + +@dataclass(frozen=True) +class PowerDefault: + """One Appendix D value: watts, and the subsection it came from.""" + + watts: float + source: str + + +_W_PER_KW = 1000.0 + + +def _catalog_default( + kind: str, + catalog: Mapping[str, Any], + model: str | None = None, + cores: int | None = None, + cabling: str | None = None, +) -> PowerDefault | None: + defaults = catalog.get("defaults", {}) + text = (model or "").lower() + if kind == "nic": + watts = defaults.get("nic_watts") + return PowerDefault(watts, "D.4") if watts is not None else None + if kind == "cpu": + if cores is None: + match = re.search("(\\d+)[\\s-]*core", text) + cores = int(match.group(1)) if match else None + architecture = ( + "arm" + if re.search( + "\\b(grace|neoverse|arm|graviton|axion|ampere|altra|agi)\\b", text + ) + else "x86" + if re.search("\\b(xeon|epyc|intel|amd|x86|x86_64)\\b", text) + else None + ) + if architecture is None or cores is None: + return None + for row in defaults.get("cpu", ()): + if row.get("architecture") == architecture and row.get( + "minimum_cores", 0 + ) <= cores <= row.get("maximum_cores", float("inf")): + return PowerDefault(row["watts"], "D.2") + return None + key = "accelerators" if kind == "accelerator" else "switches" + names = defaults.get(key, {}) + squashed = re.sub("[^a-z0-9]", "", text) + for name in sorted(names, key=len, reverse=True): + if name in squashed: + watts = names[name] if kind == "accelerator" else names[name].get(cabling) + return ( + PowerDefault(watts, "D.3" if kind == "accelerator" else "D.4") + if watts is not None + else None + ) + return None + + +class _Tally: + """Collects one figure's worth of watts, defaults and gaps while summing.""" + + def __init__(self) -> None: + self.estimated: list[str] = [] + self.gaps: list[str] = [] + self.problems: list[str] = [] + + def watts( + self, + label: str, + value: SourcedValue | None, + fallback: PowerDefault | None = None, + ) -> float | None: + """*value* in watts, or the Appendix D *fallback* where it is absent.""" + if value is not None: + watts = value.watts + if watts is None: + self.problems.append( + f"{label} is an energy figure; give value_w or value_kw" + ) + return None + if value.is_default: + self.estimated.append(f"{label} ({value.source} default)") + return watts + if fallback is not None: + self.estimated.append( + f"{label} (absent; auto-populated from {fallback.source})" + ) + return fallback.watts + self.gaps.append(f"{label} is absent and Appendix D has no default for it") + return None + + +class PowerCalculator: + def __init__(self, descriptor: SystemPower, policy: Mapping[str, Any]) -> None: + self.descriptor = descriptor + self.policy = policy + + @property + def overhead_fraction(self) -> float: + """§4.5.2's overhead fraction for the declared ``cooling``.""" + return self.policy["cooling_overhead"][self.descriptor.cooling] + + def compute( + self, cores_by_ensemble: dict[int, int] | None = None + ) -> PowerComputation: + """E.5's arithmetic, with Appendix D filling what the descriptor leaves absent.""" + cores = cores_by_ensemble or {} + out = PowerComputation(overhead_fraction=self.overhead_fraction) + tally = _Tally() + ids = [s.node_set_id for s in self.descriptor.node_sets] + if len(set(ids)) != len(ids): + out.problems.append("node_set_id values must be unique within the file") + major: dict[int, float] = {} + published: dict[int, float] = {} + for node_set in self.descriptor.node_sets: + set_major, set_published = self._add_node_set(node_set, cores, out, tally) + major[node_set.node_set_id] = set_major + published[node_set.node_set_id] = set_published + nic_per_node_w = self._add_scale_out(out, tally) + for node_set in self.descriptor.node_sets: + if self._carries_nics(node_set): + major[node_set.node_set_id] += ( + node_set.nodes_provisioned * nic_per_node_w + ) + out.major_components_w = sum(major.values()) + out.published_node_power_w = sum(published.values()) + out.other_components_w = out.overhead_fraction * out.major_components_w + out.sets = [ + SetPower( + node_set_id=s.node_set_id, + system_node_ensemble_id=s.system_node_ensemble_id, + nodes_provisioned=s.nodes_provisioned, + power_w=(1 + out.overhead_fraction) * major[s.node_set_id] + + published[s.node_set_id], + accelerators_per_node=s.components.accelerator.count_per_node + if s.components is not None and s.components.accelerator is not None + else None, + ) + for s in self.descriptor.node_sets + ] + out.problems.extend(tally.problems) + declared = self.descriptor.declared_provisioned_power + if declared is not None: + out.declared = True + watts = declared.watts + if watts is None: + out.problems.append( + "declared_provisioned_power must give value_kw or value_w" + ) + else: + out.provisioned_power_kw = round(watts / _W_PER_KW, 2) + if declared.is_default: + out.estimated.append("declared_provisioned_power (mlc_default)") + else: + out.problems.extend(tally.gaps) + out.estimated.extend(tally.estimated) + if not out.problems: + out.provisioned_power_kw = round( + out.total_system_power_w / _W_PER_KW, 2 + ) + if not out.problems and (not tally.gaps): + out.problems.extend(self._computed_disagreements(out)) + if out.problems: + out.provisioned_power_kw = None + return out + + def _add_node_set( + self, + node_set: NodeSet, + cores: dict[int, int], + out: PowerComputation, + tally: _Tally, + ) -> tuple[float, float]: + """One set's ``(major, published)`` watts. A set that cannot be costed adds 0.""" + label = f"node_sets[{node_set.node_set_id}]" + y = node_set.nodes_provisioned + if node_set.power_method == "component_sum": + if node_set.components is None: + out.problems.append(f"{label}: component_sum requires components") + return (0.0, 0.0) + per_node = self._per_node_w( + node_set.components, + cores.get(node_set.system_node_ensemble_id), + label, + tally, + ) + return (0.0 if per_node is None else y * per_node, 0.0) + if node_set.published_power is None: + out.problems.append( + f"{label}: {node_set.power_method} requires published_power" + ) + return (0.0, 0.0) + published = tally.watts(f"{label}.published_power", node_set.published_power) + if published is None: + return (0.0, 0.0) + if node_set.power_method == "published_system": + return (0.0, y * published) + n = node_set.nodes_in_published_rack + if n is None or n <= y: + out.problems.append( + f"{label}: node_scaling requires nodes_in_published_rack greater than nodes_provisioned ({y}), got {n}" + ) + return (0.0, 0.0) + return (0.0, published * (y / n)) + + def _per_node_w( + self, components: Components, cores: int | None, label: str, tally: _Tally + ) -> float | None: + """One node's CPU + accelerator + scale-up, in watts.""" + label = f"{label}.components" + parts: list[float | None] = [] + cpu, acc = (components.cpu, components.accelerator) + if components.combined_cpu_accelerator is not None: + if cpu and cpu.tdp_per_unit or (acc and acc.tdp_per_unit): + tally.problems.append( + f"{label}: combined_cpu_accelerator is mutually exclusive with cpu.tdp_per_unit and accelerator.tdp_per_unit" + ) + parts.append( + tally.watts( + f"{label}.combined_cpu_accelerator", + components.combined_cpu_accelerator, + ) + ) + else: + if cpu is None or acc is None: + tally.problems.append( + f"{label}: needs cpu and accelerator, or combined_cpu_accelerator" + ) + return None + cpu_w = tally.watts( + f"{label}.cpu.tdp_per_unit", + cpu.tdp_per_unit, + _catalog_default("cpu", self.policy, cpu.model, cores), + ) + acc_w = tally.watts( + f"{label}.accelerator.tdp_per_unit", + acc.tdp_per_unit, + _catalog_default("accelerator", self.policy, acc.model), + ) + parts.append(None if cpu_w is None else cpu.count_per_node * cpu_w) + parts.append(None if acc_w is None else acc.count_per_node * acc_w) + up = components.scale_up_network + if up is None: + tally.problems.append( + f"{label}: scale_up_network is required; use method none where there is no scale-up switch" + ) + return None + parts.append( + PowerCalculator._scale_up_w(up, f"{label}.scale_up_network", tally) + ) + if any(p is None for p in parts): + return None + return sum(p for p in parts if p is not None) + + @staticmethod + def _scale_up_w(up: ScaleUpNetwork, label: str, tally: _Tally) -> float | None: + if up.method == "none": + return 0.0 + if up.method == "declared_tdp": + if up.switch_count is None: + tally.problems.append(f"{label}: declared_tdp requires switch_count") + return None + per_switch = tally.watts(f"{label}.tdp_per_switch", up.tdp_per_switch) + return None if per_switch is None else up.switch_count * per_switch + if up.aggregate_bandwidth_tbps is None: + tally.problems.append( + f"{label}: bandwidth_estimate requires aggregate_bandwidth_tbps" + ) + return None + energy = up.energy_per_bit_pj + if energy is None: + tally.gaps.append( + f"{label}.energy_per_bit_pj is absent; D.1's references depend on the link protocol, so there is no default to apply" + ) + return None + if energy.value_pj is None: + tally.problems.append(f"{label}.energy_per_bit_pj must give value_pj") + return None + if energy.is_default: + tally.estimated.append( + f"{label}.energy_per_bit_pj ({energy.source} default)" + ) + return up.aggregate_bandwidth_tbps * 8.0 * energy.value_pj + + def _carries_nics(self, node_set: NodeSet) -> bool: + """Whether counted scale-out NICs are added to *node_set*'s power.""" + nics = self.descriptor.scale_out.nics + if not self.descriptor.scale_out.present or nics is None or (not nics.counted): + return False + return node_set.power_method == "component_sum" or bool( + nics.excluded_from_published_power + ) + + def _check_fabric_declared(self, out: PowerComputation) -> None: + """E.4: ``present`` follows from whether the nodes need a scale-out fabric.""" + nodes = sum(s.nodes_provisioned for s in self.descriptor.node_sets) + if self.descriptor.scale_out.present: + if nodes == 1: + out.warnings.append( + "scale_out.present is true for a single-node submission; E.4 makes it false there, and the switch power is counted against this node" + ) + return + if nodes == 1: + return + no_scale_up = all( + s.power_method == "component_sum" + and s.components is not None + and (s.components.scale_up_network is not None) + and (s.components.scale_up_network.method == "none") + for s in self.descriptor.node_sets + ) + if no_scale_up: + out.problems.append( + f"scale_out.present is false, but the system has {nodes} nodes and no scale-up network joins them; E.4 allows false only for a single node or nodes joined by a fabric already counted in scale_up_network" + ) + else: + out.warnings.append( + f"scale_out.present is false for {nodes} nodes; E.4 allows that only where the nodes are joined by a fabric already counted in scale_up_network, which the descriptor cannot show" + ) + + def _add_scale_out(self, out: PowerComputation, tally: _Tally) -> float: + """Add the switches to *out*; return the counted NIC watts **per node**.""" + fabric = self.descriptor.scale_out + nic_per_node_w = 0.0 + self._check_fabric_declared(out) + if not fabric.present: + return nic_per_node_w + missing = [ + name + for name in ("cabling", "required_bandwidth_tbps", "nics", "switches") + if getattr(fabric, name) is None + ] + if missing: + out.problems.append( + "scale_out.present is true but {missing} is missing".format( + missing=", ".join(missing) + ) + ) + return nic_per_node_w + assert fabric.nics is not None and fabric.switches is not None + assert fabric.required_bandwidth_tbps is not None + nodes = sum(s.nodes_provisioned for s in self.descriptor.node_sets) + nics = fabric.nics + by_formula = any( + s.power_method == "component_sum" for s in self.descriptor.node_sets + ) + if nics.counted: + if not by_formula and (not nics.excluded_from_published_power): + out.problems.append( + "scale_out.nics.counted is true, but node power comes from a published specification, which §4.5.2 assumes includes the adapters. Set counted to false, or evidence the exclusion in excluded_from_published_power" + ) + per_nic = tally.watts( + "scale_out.nics.tdp_per_nic", + nics.tdp_per_nic, + _catalog_default("nic", self.policy), + ) + if per_nic is not None: + nic_per_node_w = nics.count_per_node * per_nic + elif by_formula: + out.problems.append( + "scale_out.nics.counted is false, but node power is built with the MLC formula, which has no NIC term — §4.5.2 says the NICs MUST be included" + ) + from_nics = nodes * nics.count_per_node * nics.bandwidth_per_nic_gbps / 1000.0 + required = max(fabric.required_bandwidth_tbps, from_nics) + if fabric.required_bandwidth_tbps < from_nics - 1e-06: + out.problems.append( + f"scale_out.required_bandwidth_tbps is {fabric.required_bandwidth_tbps:g}, below the {from_nics:g} Tb/s of NIC bandwidth E.4 defines it as ({nodes} nodes × {nics.count_per_node} NICs × {nics.bandwidth_per_nic_gbps:g} Gb/s)" + ) + elif fabric.required_bandwidth_tbps > from_nics + 1e-06: + out.warnings.append( + f"scale_out.required_bandwidth_tbps is {fabric.required_bandwidth_tbps:g}, above the {from_nics:g} Tb/s the NICs carry ({nodes} nodes × {nics.count_per_node} NICs × {nics.bandwidth_per_nic_gbps:g} Gb/s)" + ) + offered = 0.0 + for index, switch in enumerate(fabric.switches): + label = f"scale_out.switches[{index}].power_per_switch" + watts = tally.watts( + label, + switch.power_per_switch, + _catalog_default( + "switch", self.policy, switch.model, cabling=fabric.cabling + ), + ) + if watts is not None: + out.scale_out_switch_power_w += switch.count * watts + offered += switch.count * switch.bandwidth_tbps + if offered < required - 1e-06: + out.problems.append( + f"scale-out switches offer {offered:g} Tb/s, below the required {required:g} Tb/s" + ) + return nic_per_node_w + + def _computed_disagreements(self, out: PowerComputation) -> list[str]: + """E.7: a submitter-supplied figure that disagrees with the recomputation.""" + problems: list[str] = [] + stated = self.descriptor.computed + if stated is not None: + ours = { + "major_components_w": out.major_components_w, + "other_components_w": out.other_components_w, + "published_node_power_w": out.published_node_power_w, + "scale_out_switch_power_w": out.scale_out_switch_power_w, + "total_system_power_w": out.total_system_power_w, + } + for name, value in ours.items(): + theirs = getattr(stated, name) + if ( + theirs is not None + and abs(theirs - value) > self.policy["computed_tolerance_w"] + ): + problems.append( + f"computed.{name} is {theirs:g}, recomputed {value:g}" + ) + if ( + stated.overhead_fraction is not None + and abs(stated.overhead_fraction - out.overhead_fraction) > 1e-09 + ): + problems.append( + f"computed.overhead_fraction is {stated.overhead_fraction:g}, but cooling {self.descriptor.cooling!r} fixes it at {out.overhead_fraction:g}" + ) + kw = out.provisioned_power_kw + stated_kw = self.descriptor.provisioned_power_kw + if ( + stated_kw is not None + and kw is not None + and (abs(stated_kw - kw) > self.policy["computed_tolerance_kw"]) + ): + problems.append( + f"provisioned_power_kw is {stated_kw:g}, recomputed {kw:.2f}" + ) + return problems diff --git a/src/inference_endpoint/validation/power/models.py b/src/inference_endpoint/validation/power/models.py new file mode 100644 index 000000000..e40ad067a --- /dev/null +++ b/src/inference_endpoint/validation/power/models.py @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Power descriptor schema and calculation results.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Literal, TypedDict + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from ..types import Cooling + +_W_PER_KW = 1000.0 + +SourceType = Literal["vendor_spec", "publication", "public_statement", "mlc_default"] +_APPENDIX_D = frozenset({"D.1", "D.2", "D.3", "D.4"}) + + +class SourcedValue(BaseModel): + """E.1: a power figure and the public, verifiable reference behind it.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + value_w: float | None = Field(default=None, ge=0) + value_kw: float | None = Field(default=None, ge=0) + value_pj: float | None = Field(default=None, ge=0) + source_type: SourceType + source: str = Field(min_length=1) + + @model_validator(mode="after") + def _one_value_and_a_real_source(self) -> SourcedValue: + values = [ + v for v in (self.value_w, self.value_kw, self.value_pj) if v is not None + ] + if len(values) != 1: + raise ValueError("give exactly one of value_w, value_kw or value_pj") + if self.source_type == "mlc_default": + if self.source not in _APPENDIX_D: + raise ValueError( + "an mlc_default source names its Appendix D subsection ({subsections}), not {source!r}".format( + subsections=", ".join(sorted(_APPENDIX_D)), source=self.source + ) + ) + elif not self.source.startswith(("https://", "http://")): + raise ValueError( + f"a {self.source_type} source must be a resolvable URL, not {self.source!r}" + ) + return self + + @property + def watts(self) -> float | None: + """The value in watts, or ``None`` for an energy-per-bit figure.""" + if self.value_w is not None: + return self.value_w + if self.value_kw is not None: + return self.value_kw * _W_PER_KW + return None + + @property + def is_default(self) -> bool: + """True for an Appendix D value, which sets the estimated-power tag.""" + return self.source_type == "mlc_default" + + +class _Open(BaseModel): + """Appendix E objects tolerate keys the schema does not name — ``notes`` and the like.""" + + model_config = ConfigDict(extra="allow", allow_inf_nan=False) + + +class Processor(_Open): + """E.3.1's ``cpu`` block. Counts are what is provisioned, not what the chassis holds.""" + + model: str | None = None + count_per_node: int = Field(ge=0) + tdp_per_unit: SourcedValue | None = None + + +class Accelerator(_Open): + """E.3.1's ``accelerator`` block.""" + + model: str | None = None + count_per_node: int = Field(ge=0) + tdp_per_unit: SourcedValue | None = None + below_rated_tdp: dict[str, object] | None = None + + +class ScaleUpNetwork(_Open): + """E.3.1's ``scale_up_network`` block, costed by one of three methods.""" + + method: Literal["declared_tdp", "bandwidth_estimate", "none"] + switch_count: int | None = Field(default=None, ge=0) + tdp_per_switch: SourcedValue | None = None + aggregate_bandwidth_tbps: float | None = Field(default=None, ge=0) + energy_per_bit_pj: SourcedValue | None = None + + +class Components(_Open): + """E.3.1: a node's power built from its parts, per node.""" + + cpu: Processor | None = None + accelerator: Accelerator | None = None + combined_cpu_accelerator: SourcedValue | None = None + scale_up_network: ScaleUpNetwork | None = None + + +class NodeSet(_Open): + """E.3: one set of identical nodes, and how its power was established.""" + + node_set_id: int + system_node_ensemble_id: int + nodes_provisioned: int = Field(gt=0) + power_method: Literal["component_sum", "published_system", "node_scaling"] + published_power: SourcedValue | None = None + nodes_in_published_rack: int | None = None + components: Components | None = None + + +class Nics(_Open): + """E.4's scale-out adapters.""" + + count_per_node: int = Field(ge=0) + bandwidth_per_nic_gbps: float = Field(ge=0) + counted: bool + tdp_per_nic: SourcedValue | None = None + excluded_from_published_power: str | None = None + + +class Switch(_Open): + """One scale-out switch model in E.4's ``switches`` array.""" + + model: str + count: int = Field(gt=0) + bandwidth_tbps: float = Field(ge=0) + power_per_switch: SourcedValue | None = None + + +class ScaleOut(_Open): + """E.4: the scale-out fabric. Everything but ``present`` is conditional on it.""" + + present: bool + cabling: Literal["passive", "active_optical"] | None = None + required_bandwidth_tbps: float | None = Field(default=None, ge=0) + nics: Nics | None = None + switches: list[Switch] | None = None + + +class Computed(_Open): + """E.5's derived arithmetic, as a submitter wrote it. The checker's values govern.""" + + major_components_w: float | None = None + overhead_fraction: float | None = None + other_components_w: float | None = None + published_node_power_w: float | None = None + scale_out_switch_power_w: float | None = None + total_system_power_w: float | None = None + + +@dataclass(frozen=True) +class SetPower: + """One node set's share of provisioned power — §4.5.3's ``P_s`` and ``N_s``.""" + + node_set_id: int + system_node_ensemble_id: int + nodes_provisioned: int + power_w: float + accelerators_per_node: int | None + + +@dataclass +class PowerComputation: + """Computed provisioned power and calculation findings.""" + + major_components_w: float = 0.0 + overhead_fraction: float = 0.0 + other_components_w: float = 0.0 + published_node_power_w: float = 0.0 + scale_out_switch_power_w: float = 0.0 + provisioned_power_kw: float | None = None + sets: list[SetPower] = field(default_factory=list) + declared: bool = False + estimated: list[str] = field(default_factory=list) + problems: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + @property + def total_system_power_w(self) -> float: + """E.5's ``total_system_power_w``: the four terms, unrounded.""" + return ( + self.major_components_w + + self.other_components_w + + self.published_node_power_w + + self.scale_out_switch_power_w + ) + + +class SystemPower(_Open): + """Parsed contents of a system's ``system_power.json`` (Appendix E.2).""" + + system_desc_id: str = Field(min_length=1) + cooling: Cooling + node_sets: list[NodeSet] = Field(min_length=1) + scale_out: ScaleOut + declared_provisioned_power: SourcedValue | None = None + computed: Computed | None = None + provisioned_power_kw: float | None = None + mlc_estimated_power: bool | None = None + notes: str | None = None + + +class PowerEvidence(TypedDict, total=False): + computation: PowerComputation + descriptor: SystemPower + problems: list[str] + requirements: Mapping[str, object] diff --git a/tests/unit/validation/test_policy_contracts.py b/tests/unit/validation/test_policy_contracts.py index 3af47c9c9..319b9bf6f 100644 --- a/tests/unit/validation/test_policy_contracts.py +++ b/tests/unit/validation/test_policy_contracts.py @@ -3,8 +3,11 @@ import pytest import yaml +from pydantic import ValidationError from inference_endpoint.validation import bundled_policy_path, load_policy +from inference_endpoint.validation.power.calculation import PowerCalculator +from inference_endpoint.validation.power.models import SystemPower pytestmark = pytest.mark.unit @@ -47,6 +50,41 @@ def test_missing_required_operand_is_rejected_when_loading(policy_dir): load_policy(policy_dir) +@pytest.mark.parametrize("cooling", ["air", "liquid", "mixed"]) +def test_cooling_modes_use_catalog_overhead(cooling): + descriptor = SystemPower.model_validate( + { + "system_desc_id": "system", + "cooling": cooling, + "node_sets": [ + { + "node_set_id": 1, + "system_node_ensemble_id": 1, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": {"model": "AMD EPYC 64-core", "count_per_node": 1}, + "accelerator": {"model": "NVIDIA B200", "count_per_node": 1}, + "scale_up_network": {"method": "none"}, + }, + } + ], + "scale_out": {"present": False}, + } + ) + catalog = load_policy(bundled_policy_path()).catalogs["power"] + computed = PowerCalculator(descriptor, catalog).compute() + assert computed.overhead_fraction == catalog["cooling_overhead"][cooling] + assert computed.provisioned_power_kw == ( + 2.02 if cooling in ("air", "mixed") else 1.75 + ) + + +def test_unknown_cooling_stays_invalid(): + with pytest.raises(ValidationError, match="cooling"): + SystemPower.model_validate({"cooling": "unknown"}) + + def test_valid_override_retains_false_and_numeric_values(policy_dir): path = policy_dir / "catalog.yaml" document = yaml.safe_load(path.read_text()) From a820fc6671b485f69a32b65bebc50de74bdc0c14 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 01:33:37 -0700 Subject: [PATCH 05/18] feat(validation): parse owned artifacts and classify submission runs --- .../validation/artifacts.py | 549 ++++++++++++++++++ .../validation/evidence/__init__.py | 10 + .../validation/evidence/accuracy.py | 122 ++++ .../validation/evidence/loaders.py | 158 +++++ .../validation/evidence/point_config.py | 166 ++++++ .../validation/evidence/point_summary.py | 143 +++++ .../validation/evidence/steady_state.py | 99 ++++ .../validation/evidence/system.py | 257 ++++++++ src/inference_endpoint/validation/results.py | 56 ++ tests/unit/validation/test_artifacts.py | 128 ++++ tests/unit/validation/test_evidence.py | 98 ++++ .../unit/validation/test_policy_contracts.py | 75 +++ tests/unit/validation/test_runtime_safety.py | 29 + 13 files changed, 1890 insertions(+) create mode 100644 src/inference_endpoint/validation/artifacts.py create mode 100644 src/inference_endpoint/validation/evidence/__init__.py create mode 100644 src/inference_endpoint/validation/evidence/accuracy.py create mode 100644 src/inference_endpoint/validation/evidence/loaders.py create mode 100644 src/inference_endpoint/validation/evidence/point_config.py create mode 100644 src/inference_endpoint/validation/evidence/point_summary.py create mode 100644 src/inference_endpoint/validation/evidence/steady_state.py create mode 100644 src/inference_endpoint/validation/evidence/system.py create mode 100644 src/inference_endpoint/validation/results.py create mode 100644 tests/unit/validation/test_artifacts.py create mode 100644 tests/unit/validation/test_evidence.py create mode 100644 tests/unit/validation/test_runtime_safety.py diff --git a/src/inference_endpoint/validation/artifacts.py b/src/inference_endpoint/validation/artifacts.py new file mode 100644 index 000000000..4bb528b3b --- /dev/null +++ b/src/inference_endpoint/validation/artifacts.py @@ -0,0 +1,549 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Artifact parsing and classification, independent of compliance execution.""" + +from __future__ import annotations + +import json +import math +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml +from pydantic import JsonValue, TypeAdapter + +from .catalogs.seeds import load_seed_sets +from .conditions import Context +from .evidence import ( + AccuracyResult, + PointConfig, + PointSummary, + SystemDescription, +) +from .evidence.loaders import ParsedArtifact, read_artifact +from .models import Policy +from .planner import PlannedCheck, plan_checks +from .power.models import PowerEvidence +from .types import Declaration, EvidenceKey, OfflineMode, Scope +from .vocabulary import CheckKind + +JsonObject = dict[str, JsonValue] +_JSON_MAPPING = TypeAdapter(JsonObject) + + +def nested(value: object, address: str, default: object = None) -> Any: + for part in address.split("."): + if isinstance(value, Mapping): + value = value.get(part, default) + else: + value = getattr(value, part, default) + if value is default: + break + return value + + +def read_mapping(path: Path) -> JsonObject: + try: + value = ( + yaml.safe_load(path.read_text()) + if path.suffix == ".yaml" + else json.loads(path.read_text()) + ) + return _JSON_MAPPING.validate_python(value) if isinstance(value, dict) else {} + except (OSError, ValueError, yaml.YAMLError): + return {} + + +def directories(path: Path) -> list[Path]: + return ( + sorted((p for p in path.iterdir() if p.is_dir()), key=lambda p: p.name) + if path.is_dir() + else [] + ) + + +@dataclass(frozen=True) +class PointEvidence: + config: ParsedArtifact[PointConfig] + summary: ParsedArtifact[PointSummary] + system: ParsedArtifact[SystemDescription] + accuracy: ParsedArtifact[AccuracyResult] + + +@dataclass +class PointDerived: + power_kw: float | None = None + + +@dataclass +class PointArtifacts: + path: Path + evidence: PointEvidence + context: Context + derived: PointDerived = field(default_factory=PointDerived) + + @classmethod + def from_evidence( + cls, + path: Path, + evidence: PointEvidence, + available: frozenset[EvidenceKey] = frozenset(), + ) -> PointArtifacts: + config, system = evidence.config, evidence.system + invalid = ( + frozenset() + if config.value is not None + else frozenset({EvidenceKey.POINT_CONFIG}) + ) + available_keys = set(available) + if config.value is not None: + available_keys.add(EvidenceKey.POINT_CONFIG) + declared = frozenset( + item for item in Declaration if item.value in config.fields + ) + for item in declared: + if config.get(item.value) is not None: + available_keys.add(EvidenceKey(item.value)) + context = Context( + id=str(path), + scope=Scope.POINT, + model_id=path.parent.name, + load_pattern=config.get("runtime_settings.load_pattern"), + offline=config.get("offline") or OfflineMode.NONE, + division=config.get("division") or system.get("division"), + declared=declared, + speculative_decoding=( + config.value.speculative_decoding is not None + or config.value.drafter is not None + ) + if config.value is not None + else None, + available=frozenset(available_keys), + invalid=invalid, + ) + return cls(path, evidence, context) + + @classmethod + def from_json( + cls, path: Path, config: JsonObject, summary: JsonObject, system: JsonObject + ) -> PointArtifacts: + return cls.from_evidence( + path, + PointEvidence( + ParsedArtifact.from_json( + config, PointConfig, path / "point.yaml", "point-config-valid" + ), + ParsedArtifact.from_json( + summary, + PointSummary, + path / "result_summary.json", + "result-file-valid", + ), + ParsedArtifact.from_json( + system, + SystemDescription, + path / "system_desc.json", + "system-description-valid", + ), + ParsedArtifact(None, frozenset(), (), False), + ), + ) + + @property + def config(self) -> PointConfig | None: + return self.evidence.config.value + + @property + def summary(self) -> PointSummary | None: + return self.evidence.summary.value + + @property + def system(self) -> SystemDescription | None: + return self.evidence.system.value + + @property + def accuracy(self) -> AccuracyResult | None: + return self.evidence.accuracy.value + + @property + def curve(self) -> Path: + return self.path.parent + + def get_config(self, address: str, default: object = None) -> Any: + return self.evidence.config.get(address, default) + + def get_summary(self, address: str, default: object = None) -> Any: + return self.evidence.summary.get(address, default) + + def get_system(self, address: str, default: object = None) -> Any: + return self.evidence.system.get(address, default) + + @property + def concurrency(self) -> int | None: + value = self.get_config("concurrency") + if isinstance(value, int) and not isinstance(value, bool) and value > 0: + return value + return int(self.path.name[1:]) if self.path.name[1:].isdigit() else None + + @property + def throughput(self) -> float | None: + duration = self.get_summary("duration_ns") + tokens = self.get_summary("output_sequence_lengths.total") + if duration is None or tokens is None or duration <= 0: + return None + return tokens / (duration / 1e9) + + +@dataclass +class ArtifactIndex: + points: dict[str, PointArtifacts] = field(default_factory=dict) + subjects: list[Context] = field(default_factory=list) + curves: dict[str, list[str]] = field(default_factory=dict) + systems: dict[str, list[str]] = field(default_factory=dict) + + +@dataclass +class CatalogEvidence: + values: dict[str, object] = field(default_factory=dict) + errors: dict[str, str] = field(default_factory=dict) + + +@dataclass +class DerivedArtifacts: + regions: dict[str, dict[str, tuple[int | float, int | float]]] = field( + default_factory=dict + ) + region_errors: dict[str, str] = field(default_factory=dict) + power: dict[str, PowerEvidence] = field(default_factory=dict) + + +@dataclass +class Artifacts: + root: Path + policy: Policy + index: ArtifactIndex = field(default_factory=ArtifactIndex) + catalogs: CatalogEvidence = field(default_factory=CatalogEvidence) + derived: DerivedArtifacts = field(default_factory=DerivedArtifacts) + + def members(self, check: PlannedCheck) -> list[PointArtifacts]: + subject = check.subject + if subject.scope is Scope.POINT: + return [self.index.points[subject.id]] + ids = self.index.curves.get( + subject.id, self.index.systems.get(subject.id, list(self.index.points)) + ) + if check.selected_members is not None: + ids = [id for id in ids if id in check.selected_members] + return [self.index.points[id] for id in ids] + + def resolve( + self, + address: str | None, + check: PlannedCheck, + point: PointArtifacts | None = None, + ) -> Any: + if not isinstance(address, str): + return address + if address == "catalogs.approved_sped_decode_heads": + return self.catalogs.values.get("approved_sped_decode_heads") + if address.startswith("catalogs."): + return nested(self.policy.catalogs, address[9:]) + if address == "enrollment.models": + return self.policy.models + if address.startswith("context.cohorts."): + return nested(self.catalogs.values, address[16:]) + if address.startswith("model."): + return nested( + self.policy.catalogs, + "models." + str(check.subject.model_id) + "." + address[6:], + ) + path = Path(check.subject.id) + if address == "submission.root": + return self.root + if address == "submission.system_directories": + return directories(self.root / "results") + if address == "submission.implementation_directories": + return directories(self.root / "src") + if address == "implementation.readme": + return ( + [p for p in path.iterdir() if p.name.lower() == "readme.md"] + if path.is_dir() + else [] + ) + if address == "system.model_directories": + return directories(path) + members = self.members(check) + if address == "curve.point_directories": + return [p.path for p in members] + if address == "curve.parsed_points": + return [p for p in members if p.config is not None] + if address == "curve.model_directory_name": + return path.name + if address == "curve.first_declared_model_name": + return next( + ( + p.get_config("model_name") + for p in members + if p.get_config("model_name") + ), + None, + ) + if address == "curve.max_supported_concurrency": + return next( + ( + p.get_system("max_supported_concurrency") + for p in members + if p.get_system("max_supported_concurrency") is not None + ), + None, + ) + if point is None and check.subject.scope is Scope.POINT: + point = self.index.points[check.subject.id] + if point is None: + return [self.resolve(address, check, p) for p in members] + if address in ( + "point.yaml", + "point.system_desc.json", + "point.result_summary.json", + "result_summary.json", + ): + return ( + point.path + / { + "point.yaml": "point.yaml", + "point.system_desc.json": "system_desc.json", + "point.result_summary.json": "result_summary.json", + "result_summary.json": "result_summary.json", + }[address] + ) + if address == "point.directory": + return point.path + if address == "point.directory_concurrency": + return int(point.path.name[1:]) + if address == "point.power_kw": + return point.derived.power_kw + if address == "point.system_description": + return ( + point.system.model_dump(mode="json", exclude_unset=True) + if point.system is not None + else None + ) + if address.startswith("point.system_description."): + return point.get_system(address[25:]) + if address.startswith("point."): + return point.get_config(address[6:]) + if address == "result_summary.raw_derived_system_tps": + return point.throughput + if ( + address == "result_summary.tpot.percentiles.90" + and point.summary is not None + ): + return point.summary.tpot.percentiles.get("90") + if address.startswith("result_summary."): + return point.get_summary(address[15:]) + if address.startswith("accuracy."): + configured = point.get_config(address) + if configured is not None: + return configured + if point.accuracy is None: + return None + dataset, _, field = address[9:].partition(".") + entry: JsonObject = next( + ( + entry + for name, entry in point.accuracy.root.items() + if name.split("::")[0] + in ( + {"inline", "agentic_inference_inline", "agentic_combined"} + if dataset == "inline" + else {dataset} + ) + ), + {}, + ) + return nested(entry, field) + return None + + def region(self, point: PointArtifacts) -> str | None: + regions = self.derived.regions.get(str(point.curve), {}) + return next( + ( + name + for name, (lo, hi) in regions.items() + if point.concurrency is not None and lo <= point.concurrency <= hi + ), + None, + ) + + +def load_artifacts(root: Path, policy: Policy) -> Artifacts: + artifacts = Artifacts(root, policy) + try: + artifacts.catalogs.values["seed_sets"] = load_seed_sets() + except (ValueError, OSError) as exc: + artifacts.catalogs.errors["seed_sets"] = str(exc) + artifacts.catalogs.values["approved_sped_decode_heads"] = policy.catalogs.get( + "approved_sped_decode_heads" + ) + for system_dir in directories(root / "results"): + system_ids = [] + for curve_dir in directories(system_dir): + point_ids = [] + for path in directories(curve_dir): + if not (path.name.startswith("r") and path.name[1:].isdigit()): + continue + accuracy_path = path / "accuracy_results.json" + evidence = PointEvidence( + read_artifact( + path / "point.yaml", PointConfig, "point-config-valid" + ), + read_artifact( + path / "result_summary.json", PointSummary, "result-file-valid" + ), + read_artifact( + path / "system_desc.json", + SystemDescription, + "system-description-valid", + ), + read_artifact( + accuracy_path + if accuracy_path.exists() + else path / "results.json", + AccuracyResult, + "accuracy-valid", + accuracy_scores=not accuracy_path.exists(), + ), + ) + available = set() + if "seed_sets" in artifacts.catalogs.values: + available.add(EvidenceKey.SEED_CATALOG) + if "approved_sped_decode_heads" in artifacts.catalogs.values: + available.add(EvidenceKey.APPROVED_SPED_DECODE_HEADS) + point = PointArtifacts.from_evidence( + path, evidence, frozenset(available) + ) + artifacts.index.points[str(path)] = point + point_ids.append(str(path)) + artifacts.index.curves[str(curve_dir)] = point_ids + system_ids.extend(point_ids) + members = tuple(artifacts.index.points[id].context for id in point_ids) + first = members[0] if members else None + artifacts.index.subjects.append( + Context( + id=str(curve_dir), + scope=Scope.CURVE, + model_id=curve_dir.name, + division=first.division if first else None, + members=members, + ) + ) + _compute_regions(artifacts, curve_dir, point_ids) + artifacts.index.systems[str(system_dir)] = system_ids + first_point = artifacts.index.points[system_ids[0]] if system_ids else None + declarations = ( + frozenset({Declaration.POWER_DESCRIPTOR}) + if (system_dir / "system_power.json").exists() + else frozenset() + ) + artifacts.index.subjects.append( + Context( + id=str(system_dir), + scope=Scope.SYSTEM, + division=first_point.context.division if first_point else None, + declared=declarations, + ) + ) + # Refresh collection members after derived evidence has been computed. + artifacts.index.subjects = [ + s.model_copy( + update={ + "members": tuple( + artifacts.index.points[id].context + for id in artifacts.index.curves[s.id] + ), + "available": frozenset({EvidenceKey.COMPUTED_REGIONS}) + if s.id in artifacts.derived.regions + else frozenset(), + } + ) + if s.scope is Scope.CURVE + else s + for s in artifacts.index.subjects + ] + artifacts.index.subjects.extend(p.context for p in artifacts.index.points.values()) + artifacts.index.subjects.append(Context(id=str(root), scope=Scope.SUBMISSION)) + artifacts.index.subjects.append(Context(id=str(root), scope=Scope.VALIDATOR)) + artifacts.index.subjects.extend( + Context(id=str(p), scope=Scope.IMPLEMENTATION) + for p in directories(root / "src") + ) + return artifacts + + +def _compute_regions(artifacts: Artifacts, curve: Path, ids: list[str]) -> None: + all_points = [artifacts.index.points[id] for id in ids] + catalog = artifacts.policy.catalogs.get("regions", {}) + subject = next(s for s in artifacts.index.subjects if s.id == str(curve)) + plans = [ + check + for check in plan_checks(artifacts.policy, [subject]).ready + if check.rule.kind is CheckKind.REGION_BASIS + ] + if not plans: + return + effective_clamps = {check.rule.requirements["upper_clamp"] for check in plans} + if len(effective_clamps) != 1: + artifacts.derived.region_errors[str(curve)] = ( + "Conflicting region-basis partitions cannot share one set of boundaries" + ) + return + clamp = effective_clamps.pop() + selected_ids = { + str(point.path) for check in plans for point in artifacts.members(check) + } + points = [point for point in all_points if str(point.path) in selected_ids] + cmax = next( + ( + p.get_system("max_supported_concurrency") + for p in points + if p.get_system("max_supported_concurrency") is not None + ), + None, + ) + concurrencies = [p.concurrency for p in points if p.concurrency is not None] + if ( + not concurrencies + or not isinstance(cmax, (int, float)) + or isinstance(cmax, bool) + or not math.isfinite(cmax) + or cmax <= clamp + ): + return + cmin = min(min(concurrencies), clamp) + if cmin < 1: + return + partitions = catalog.get("logarithmic_partitions", 3) + if ( + partitions != 3 + or catalog.get("rounding", "half_even") != "half_even" + or catalog.get("margin_rounding", "ceiling") != "ceiling" + ): + return + interval = math.log2(cmax - cmin) / partitions + low = round(cmin + 2**interval) + medium = round(cmin + 2 ** (2 * interval)) + artifacts.derived.regions[str(curve)] = { + "low_latency": (1, cmin), + "low_concurrency": (cmin + 1, low), + "med_concurrency": (low + 1, medium), + "high_concurrency": (medium + 1, cmax), + "margin": (cmax + 1, math.ceil(catalog.get("margin_multiplier", 1.1) * cmax)), + } + for point in all_points: + point.context = point.context.model_copy( + update={ + "available": point.context.available | {EvidenceKey.COMPUTED_REGIONS} + } + ) diff --git a/src/inference_endpoint/validation/evidence/__init__.py b/src/inference_endpoint/validation/evidence/__init__.py new file mode 100644 index 000000000..6aae2e060 --- /dev/null +++ b/src/inference_endpoint/validation/evidence/__init__.py @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Typed submission artifacts, without embedded compliance checks.""" + +from .accuracy import AccuracyResult +from .point_config import PointConfig +from .point_summary import PointSummary +from .system import SystemDescription + +__all__ = ["AccuracyResult", "PointConfig", "PointSummary", "SystemDescription"] diff --git a/src/inference_endpoint/validation/evidence/accuracy.py b/src/inference_endpoint/validation/evidence/accuracy.py new file mode 100644 index 000000000..df69f54c4 --- /dev/null +++ b/src/inference_endpoint/validation/evidence/accuracy.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Native dataset accuracy entries and metric normalization.""" + +from __future__ import annotations + +import math +from typing import cast + +from pydantic import JsonValue, RootModel, model_validator + + +class AccuracyResult(RootModel[dict[str, dict[str, JsonValue]]]): + """Parsed dataset accuracy entries.""" + + @model_validator(mode="before") + @classmethod + def _read_native_report(cls, data: object) -> object: + """Accept native reports and index their entries without changing the file.""" + if isinstance(data, dict) and "accuracy_scores" in data: + data = data["accuracy_scores"] + if not isinstance(data, (dict, list)): + raise ValueError( + "accuracy_scores must be a mapping of datasets or a list of entries" + ) + if not isinstance(data, list): + return data + indexed: dict[str, dict[str, JsonValue]] = {} + for entry in data: + if not isinstance(entry, dict): + raise ValueError("each accuracy_scores entry must be an object") + name = entry.get("dataset_name") + if not isinstance(name, str) or not name.strip(): + raise ValueError( + "each accuracy_scores entry needs a non-empty dataset_name" + ) + if name in indexed: + raise ValueError(f"dataset {name!r} appears more than once") + if "score" not in entry: + raise ValueError(f"dataset {name!r} has no score") + normalized = dict(entry) + for native, canonical in ( + ("unit_samples", "num_samples"), + ("num_repeats", "n_repeats"), + ): + if native in entry: + if canonical in entry and entry[canonical] != entry[native]: + raise ValueError( + f"dataset {name!r} gives {native} and {canonical} different values" + ) + normalized[canonical] = entry[native] + indexed[name] = normalized + return indexed + + _META_KEYS = frozenset( + { + "dataset_name", + "num_samples", + "status", + "extractor", + "ground_truth_column", + "n_repeats", + "complete", + "unit_samples", + "total_samples", + "num_repeats", + "duration_s", + "dataset_type", + "response_counts", + "extras", + "eval_method", + "evaluated_instance_count", + } + ) + + @classmethod + def _metric_values(cls, entry: dict[str, JsonValue]) -> dict[str, JsonValue]: + if "score" in entry: + score = entry["score"] + return score if isinstance(score, dict) else {"score": score} + return { + name: value for name, value in entry.items() if name not in cls._META_KEYS + } + + @model_validator(mode="after") + def _validate_supplied_scores(self) -> AccuracyResult: + for dataset, entry in self.root.items(): + scores = self._metric_values(entry) + if "score" in entry and not scores: + raise ValueError(f"dataset {dataset!r} supplies an empty score mapping") + normalized: dict[str, JsonValue] = {} + for metric, value in scores.items(): + message = f"dataset {dataset!r} metric {metric!r} must be a finite numeric score" + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + raise ValueError(message) + try: + number = float(value) + except (ValueError, OverflowError) as error: + raise ValueError(message) from error + if not math.isfinite(number): + raise ValueError(message) + normalized[metric] = number + if "score" in entry: + entry["score"] = ( + normalized + if isinstance(entry["score"], dict) + else normalized["score"] + ) + else: + entry.update(normalized) + return self + + def metric_scores(self) -> dict[str, dict[str, float]]: + """Return validated numeric metrics indexed by dataset.""" + return { + dataset: { + metric: cast(float, value) + for metric, value in self._metric_values(entry).items() + } + for dataset, entry in self.root.items() + if self._metric_values(entry) + } diff --git a/src/inference_endpoint/validation/evidence/loaders.py b/src/inference_endpoint/validation/evidence/loaders.py new file mode 100644 index 000000000..a29b47ab6 --- /dev/null +++ b/src/inference_endpoint/validation/evidence/loaders.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Read once, parse evidence, and retain field presence rather than input documents.""" + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Any, Generic, TypeVar + +import yaml +from pydantic import BaseModel, ValidationError + +from ..results import CheckResult +from .accuracy import AccuracyResult +from .point_config import PointConfig +from .point_summary import PointSummary +from .system import SystemDescription + + +def supplied_fields(value: object, prefix: str = "") -> frozenset[str]: + fields = set() + if isinstance(value, Mapping): + for key, child in value.items(): + name = f"{prefix}.{key}" if prefix else str(key) + fields.add(name) + fields.update(supplied_fields(child, name)) + return frozenset(fields) + + +def schema_errors( + error: ValidationError, rule: str, path: Path +) -> tuple[CheckResult, ...]: + return tuple( + CheckResult( + rule=rule, + key=item["type"], + message=f"{'.'.join(map(str, item['loc'])) or path.name}: {item['msg']}", + path=path, + ) + for item in error.errors(include_url=False) + ) + + +def evidence_value(value: object, address: str, default: object = None) -> Any: + for name in address.split("."): + if isinstance(value, BaseModel): + # Submitted aliases take precedence over similarly named computed properties. + name = next( + ( + field + for field, info in type(value).model_fields.items() + if info.alias == name + ), + name, + ) + value = getattr(value, name, default) + elif isinstance(value, Mapping): + value = value.get(name, default) + else: + return default + if isinstance(value, BaseModel): + return value.model_dump(mode="json", exclude_unset=True) + if isinstance(value, (list, tuple)): + return [ + item.model_dump(mode="json", exclude_unset=True) + if isinstance(item, BaseModel) + else item + for item in value + ] + return value.value if isinstance(value, Enum) else value + + +T = TypeVar("T", bound=BaseModel) + + +@dataclass(frozen=True) +class ParsedArtifact(Generic[T]): # noqa: UP046 - supported by the repository type checker + value: T | None + fields: frozenset[str] + errors: tuple[CheckResult, ...] + present: bool + + @classmethod + def from_json( + cls, data: object, model: type[T], path: Path, rule: str + ) -> "ParsedArtifact[T]": + fields = supplied_fields(data) + try: + return cls(model.model_validate(data), fields, (), True) + except ValidationError as error: + return cls(None, fields, schema_errors(error, rule, path), True) + + def get(self, address: str, default: object = None) -> Any: + return ( + evidence_value(self.value, address, default) + if address in self.fields + else default + ) + + +def read_artifact( # noqa: UP047 - supported by the repository type checker + path: Path, model: type[T], rule: str, *, accuracy_scores: bool = False +) -> ParsedArtifact[T]: + try: + text = path.read_text(encoding="utf-8") + data = ( + yaml.safe_load(text) + if path.suffix in {".yaml", ".yml"} + else json.loads(text) + ) + except (OSError, UnicodeError, json.JSONDecodeError, yaml.YAMLError) as error: + if accuracy_scores: + # Result-file checks own failures in the inline accuracy source. + return ParsedArtifact(None, frozenset(), (), False) + finding = CheckResult( + rule=rule, + key="artifact-unreadable", + message=f"Cannot parse {path.name}: {error}", + path=path, + ) + return ParsedArtifact(None, frozenset(), (finding,), path.exists()) + if accuracy_scores: + data = data.get("accuracy_scores") if isinstance(data, Mapping) else None + if data is None or data == {} or data == []: + return ParsedArtifact(None, frozenset(), (), False) + data = {"accuracy_scores": data} + return ParsedArtifact.from_json(data, model, path, rule) + + +def load_point_config(path: Path) -> tuple[PointConfig | None, list[CheckResult]]: + parsed = read_artifact(path, PointConfig, "point-config-valid") + return parsed.value, list(parsed.errors) + + +def load_result_summary(path: Path) -> tuple[PointSummary | None, list[CheckResult]]: + parsed = read_artifact(path, PointSummary, "result-file-valid") + return parsed.value, list(parsed.errors) + + +def load_system_description( + path: Path, +) -> tuple[SystemDescription | None, list[CheckResult]]: + parsed = read_artifact(path, SystemDescription, "system-description-valid") + return parsed.value, list(parsed.errors) + + +def load_accuracy_result(path: Path) -> tuple[AccuracyResult | None, list[CheckResult]]: + parsed = read_artifact(path, AccuracyResult, "accuracy-valid") + return parsed.value, list(parsed.errors) + + +def load_accuracy_scores( + path: Path, +) -> tuple[AccuracyResult | None, list[CheckResult], bool]: + parsed = read_artifact(path, AccuracyResult, "accuracy-valid", accuracy_scores=True) + return parsed.value, list(parsed.errors), parsed.present diff --git a/src/inference_endpoint/validation/evidence/point_config.py b/src/inference_endpoint/validation/evidence/point_config.py new file mode 100644 index 000000000..6cd20a452 --- /dev/null +++ b/src/inference_endpoint/validation/evidence/point_config.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Typed point configuration and nested declarations.""" + +from __future__ import annotations + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + JsonValue, + StrictBool, + StrictInt, + model_validator, +) + +from inference_endpoint.config.schema import LoadPatternType + +from ..types import Division, OfflineMode +from .steady_state import SteadyState + + +class WarmupSpec(BaseModel): + """Warmup procedure declaration required by §6.3.3.""" + + model_config = ConfigDict(extra="ignore") + duration_s: float = Field(ge=0) + requests_issued: int = Field(ge=0) + requests_completed: int = Field(ge=0) + data_source: str = Field(min_length=1) + concurrency: int = Field(ge=0) + initialization_steps: list[str] = Field(default_factory=list) + logs_retained: bool | None = None + link_logs: str | None = None + + @property + def is_disabled(self) -> bool: + """Whether the declaration records no warmup duration or requests.""" + return ( + self.duration_s == 0 + and self.requests_issued == 0 + and (self.requests_completed == 0) + ) + + @model_validator(mode="after") + def _check_completed_le_issued(self) -> WarmupSpec: + if self.requests_completed > self.requests_issued: + raise ValueError( + f"warmup requests_completed ({self.requests_completed}) exceeds requests_issued ({self.requests_issued})" + ) + if self.concurrency == 0 and (not self.is_disabled): + raise ValueError( + "warmup concurrency must be positive when duration or requests are nonzero" + ) + return self + + +class NodesUsed(BaseModel): + """One entry of §8.3's ``nodes_used``: the nodes of one type a point engages.""" + + model_config = ConfigDict(extra="ignore") + system_node_ensemble_id: int + nodes: int + + +class DpShortfall(BaseModel): + """§8.3's ``dp_shortfall``: a point run at fewer replicas than §4.5.3's formula gives.""" + + model_config = ConfigDict(extra="ignore") + dp_actual: int + dp_formula: int + reason: str = Field(min_length=1) + + +class CheckpointDeclaration(BaseModel): + repository: str | None = None + revision: str | None = None + + +class DecodeHeadDeclaration(CheckpointDeclaration): + weight_checksum: str | None = None + + +class AgenticSettings(BaseModel): + enable_salt: StrictBool | None = None + inject_tool_delay: StrictBool | None = None + stop_issuing_on_first_user_complete: StrictBool | None = None + num_trajectories_to_issue: StrictInt | None = None + + +class RuntimeSettings(BaseModel): + """Runtime settings declared in a measurement point's ``point.yaml``.""" + + model_config = ConfigDict(extra="ignore") + load_pattern: LoadPatternType = LoadPatternType.CONCURRENCY + min_duration_ms: int | None = None + min_sample_count: int | None = None + stream_all_chunks: bool = True + + class Runtime(BaseModel): + """RNG seed configuration bound to a published seed set (§4.6).""" + + model_config = ConfigDict(extra="ignore") + scheduler_rng_seed: int | None = None + sample_index_rng_seed: int | None = None + model_seed: int | None = None + scheduler_random_seed: int | None = None + dataloader_random_seed: int | None = None + + @model_validator(mode="after") + def _lift_legacy_seed_names(self) -> RuntimeSettings.Runtime: + """Fill canonical seed fields from aliases when canonical values are absent.""" + if ( + self.scheduler_rng_seed is None + and self.scheduler_random_seed is not None + ): + self.scheduler_rng_seed = self.scheduler_random_seed + if ( + self.sample_index_rng_seed is None + and self.dataloader_random_seed is not None + ): + self.sample_index_rng_seed = self.dataloader_random_seed + return self + + class WarmupLoadgen(BaseModel): + """Loadgen warmup options.""" + + model_config = ConfigDict(extra="ignore") + salt: bool | None = None + + runtime: Runtime + agentic_inference: AgenticSettings | None = None + warmup: WarmupLoadgen | None = None + + +class PointConfig(BaseModel): + """Parsed contents of a Pareto point's ``point.yaml`` (§8.3).""" + + model_config = ConfigDict(extra="ignore") + concurrency: int + region: str | None = None + dataset: str = "" + runtime_settings: RuntimeSettings + warmup: WarmupSpec | None = None + division: Division | None = None + max_supported_concurrency: int | None = None + model_name: str | None = None + model_precision: str | None = None + link_to_model: str | None = None + link_to_model_transformation: str | None = None + model_notes: str | None = None + dataset_name: str | None = None + dataset_type: str | None = None + dataset_link: str | None = None + offline: OfflineMode | None = None + steady_state: SteadyState | None = None + speculative_decoding: DecodeHeadDeclaration | None = None + drafter: DecodeHeadDeclaration | None = None + checkpoint: CheckpointDeclaration | None = None + accuracy: dict[str, dict[str, JsonValue]] | None = None + nodes_used: list[NodesUsed] | None = None + dp_shortfall: DpShortfall | None = None + shared_src: str | None = None + shared_docs: str | None = None + seed_set: str | None = None + target_cohort: str | None = None diff --git a/src/inference_endpoint/validation/evidence/point_summary.py b/src/inference_endpoint/validation/evidence/point_summary.py new file mode 100644 index 000000000..e104477b4 --- /dev/null +++ b/src/inference_endpoint/validation/evidence/point_summary.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import math + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + FiniteFloat, + computed_field, + field_validator, +) + + +class PercentileStats(BaseModel): + """Summary statistics dict produced by the endpoints ``compute_summary()`` helper.""" + + model_config = ConfigDict(extra="ignore") + total: float = 0.0 + percentiles: dict[str, float] = Field(default_factory=dict) + + @field_validator("percentiles") + @classmethod + def _normalize_percentile_keys(cls, values: dict[str, float]) -> dict[str, float]: + """Treat native decimal keys and integer keys as the same percentile.""" + normalized: dict[str, float] = {} + for key, value in values.items(): + try: + percentile = float(key) + except ValueError: + percentile = math.nan + if not math.isfinite(percentile) or not 0 <= percentile <= 100: + raise ValueError(f"{key!r} is not a percentile between 0 and 100") + canonical = ( + str(int(percentile)) if percentile.is_integer() else str(percentile) + ) + if canonical in normalized and normalized[canonical] != value: + raise ValueError( + f"two keys for percentile {canonical} give different values" + ) + normalized[canonical] = value + return normalized + + +class FullRunLengths(BaseModel): + output_sequence_lengths: LengthAverage + + +class LengthAverage(BaseModel): + avg: FiniteFloat | None = None + + +class PointSummary(BaseModel): + """Parsed contents of a measurement point's ``result_summary.json``.""" + + model_config = ConfigDict(extra="ignore") + git_sha: str | None = None + system_tps_per_kw: FiniteFloat | None = None + output_sequence_lengths_full_run: FullRunLengths | None = None + tps_per_user: FiniteFloat | None = None + reported_system_tps: FiniteFloat | None = Field( + default=None, alias="system_tps", exclude=True + ) + reported_e2e_avg_interactivity: FiniteFloat | None = Field( + default=None, alias="e2e_avg_interactivity", exclude=True + ) + n_samples_issued: int = 0 + n_samples_completed: int + n_samples_failed: int = 0 + duration_ns: float + ttft: PercentileStats = Field(default_factory=PercentileStats) + tpot: PercentileStats = Field(default_factory=PercentileStats) + output_sequence_lengths: PercentileStats = Field(default_factory=PercentileStats) + output_tokens_per_turn_total: float | None = None + e2e_turn_time_seconds_total: float | None = None + + @computed_field # type: ignore[misc] + @property + def duration_ms(self) -> float: + """Measurement duration in milliseconds.""" + return self.duration_ns / 1000000 + + @computed_field # type: ignore[misc] + @property + def sample_count(self) -> int: + """Alias for ``n_samples_completed``.""" + return self.n_samples_completed + + @computed_field # type: ignore[misc] + @property + def total_output_tokens(self) -> int: + """Total output tokens from ``output_sequence_lengths.total``.""" + return int(self.output_sequence_lengths.total) + + @computed_field # type: ignore[misc] + @property + def elapsed_duration_seconds(self) -> float: + """Measurement duration in seconds.""" + return self.duration_ns / 1000000000.0 + + @computed_field # type: ignore[misc] + @property + def system_tps(self) -> float: + """System-wide tokens per second: ``total_output_tokens / elapsed_s``.""" + elapsed_s = self.elapsed_duration_seconds + return self.total_output_tokens / elapsed_s if elapsed_s > 0 else 0.0 + + @computed_field # type: ignore[misc] + @property + def ttft_p50_ms(self) -> float: + """Median time to first token in milliseconds.""" + return self.ttft.percentiles.get("50", 0.0) / 1000000 + + @computed_field # type: ignore[misc] + @property + def ttft_p90_ms(self) -> float: + """90th-percentile time to first token in milliseconds (§4.1).""" + return self.ttft.percentiles.get("90", 0.0) / 1000000 + + @computed_field # type: ignore[misc] + @property + def ttft_p95_ms(self) -> float: + """95th-percentile time to first token in milliseconds.""" + return self.ttft.percentiles.get("95", 0.0) / 1000000 + + @computed_field # type: ignore[misc] + @property + def tpot_p90_ms(self) -> float | None: + """90th-percentile time per output token in milliseconds (§9.1).""" + raw = self.tpot.percentiles.get("90") + return None if raw is None else raw / 1000000 + + @computed_field # type: ignore[misc] + @property + def e2e_avg_interactivity(self) -> float | None: + """§4.1: ``sum(output_tokens_per_turn) / sum(e2e_turn_time_seconds)``.""" + tokens = self.output_tokens_per_turn_total + seconds = self.e2e_turn_time_seconds_total + if tokens is None or seconds is None or seconds <= 0: + return None + return tokens / seconds diff --git a/src/inference_endpoint/validation/evidence/steady_state.py b/src/inference_endpoint/validation/evidence/steady_state.py new file mode 100644 index 000000000..6ddb35fd1 --- /dev/null +++ b/src/inference_endpoint/validation/evidence/steady_state.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from enum import StrEnum + +from pydantic import BaseModel, ConfigDict, Field + + +class SteadyStateStatus(StrEnum): + """Coverage status controlling the reported metric basis.""" + + WINDOWABLE = "windowable" + INSUFFICIENT_DURATION = "insufficient_duration" + INSUFFICIENT_PASSES = "insufficient_passes" + PARTIAL_DATASET = "partial_dataset" + + +class SteadyStateVerdict(StrEnum): + """Canonical detector verdict with explicit input aliases.""" + + STEADY_STATE = "steady_state" + DRIFTING_UP = "drifting_up" + DRIFTING_DOWN = "drifting_down" + ANOMALY = "anomaly" + NOT_FOUND = "not_found" + + @classmethod + def _missing_(cls, value: object) -> SteadyStateVerdict | None: + aliases = {"STEADY STATE": cls.STEADY_STATE, "not found": cls.NOT_FOUND} + return aliases.get(value) if isinstance(value, str) else None + + @property + def is_drifting(self) -> bool: + """Whether the verdict requires drift reporting rather than a point estimate.""" + return self in (self.DRIFTING_UP, self.DRIFTING_DOWN) + + +class MetricState(StrEnum): + """Canonical state of a gating metric.""" + + PLATEAU = "plateau" + DRIFTING_UP = "drifting_up" + DRIFTING_DOWN = "drifting_down" + + @classmethod + def _missing_(cls, value: object) -> MetricState | None: + aliases = { + "Plateau": cls.PLATEAU, + "Drifting Up": cls.DRIFTING_UP, + "Drifting Down": cls.DRIFTING_DOWN, + } + return aliases.get(value) if isinstance(value, str) else None + + @property + def is_drifting(self) -> bool: + """Whether this metric has an upward or downward trend.""" + return self in (self.DRIFTING_UP, self.DRIFTING_DOWN) + + +class SteadyStateWindow(BaseModel): + """The detected window's extent (§4.4).""" + + model_config = ConfigDict(extra="ignore") + n_super_passes: int | None = None + super_pass_start: int | None = None + super_pass_end: int | None = None + super_pass_size: int | None = None + n_samples: int | None = None + duration_s: float | None = None + + @property + def super_passes(self) -> int | None: + """How many super-passes the window spans, or None when unstated.""" + if self.super_pass_start is None or self.super_pass_end is None: + return None + return self.super_pass_end - self.super_pass_start + 1 + + +class SteadyState(BaseModel): + """A point's §4.4 reporting block.""" + + model_config = ConfigDict(extra="ignore") + n_super_passes: int | None = None + status: SteadyStateStatus | None = None + verdict: SteadyStateVerdict | None = None + window: SteadyStateWindow = Field(default_factory=SteadyStateWindow) + state: dict[str, MetricState] = Field(default_factory=dict) + anomaly: object | None = None + + @property + def is_official(self) -> bool: + """True when §4.4 makes the windowed metrics this point's official result.""" + return self.status == SteadyStateStatus.WINDOWABLE + + @property + def drifting_metrics(self) -> list[str]: + """Gating metrics that are drifting rather than flat.""" + return sorted((name for name, state in self.state.items() if state.is_drifting)) diff --git a/src/inference_endpoint/validation/evidence/system.py b/src/inference_endpoint/validation/evidence/system.py new file mode 100644 index 000000000..8f2e01f9c --- /dev/null +++ b/src/inference_endpoint/validation/evidence/system.py @@ -0,0 +1,257 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from enum import StrEnum +from typing import Annotated + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + StrictFloat, + StringConstraints, + field_validator, + model_validator, +) + +from ..types import Division + + +class SystemAvailabilityStatus(StrEnum): + """System availability status (§8.2).""" + + AVAILABLE = "Available" + PREVIEW = "Preview" + RDI = "RDI" + + +_ACCELERATOR_FIELDS = ( + "accelerator_model_name", + "accelerators_per_node", + "accelerator_memory_capacity", + "accelerator_memory_type", + "accelerator_interconnect", + "accelerator_host_interconnect", +) + + +class AcceleratorInfo(BaseModel): + """One accelerator configuration within a node type (§8.2.1).""" + + model_config = ConfigDict(extra="ignore") + accelerator_model_name: str | None = None + accelerators_per_node: int | None = None + accelerator_memory_capacity: str | None = None + accelerator_memory_type: str | None = None + accelerator_interconnect: str | None = None + accelerator_host_interconnect: str | None = None + + +class NodeType(BaseModel): + """Per-node hardware and software configuration (§8.2.1).""" + + model_config = ConfigDict(extra="ignore") + system_node_ensemble_id: int | None = None + number_of_nodes: int | None = None + host_processor_model_name: str | None = None + host_processors_per_node: int | None = None + host_processor_core_count: int | None = None + host_processor_vcpu_count: int | None = None + host_memory_capacity: str | None = None + host_memory_configuration: str + accelerator_info: list[AcceleratorInfo] = Field(default_factory=list) + host_network_card_count: str + host_networking: str | None = None + host_storage_capacity: str | None = None + host_storage_type: str | None = None + other_hardware: str | None = None + hw_notes: str | None = None + cooling: str | None = None + inference_backend: str | None = None + driver: str + operating_system: str | None = None + filesystem: str + container_link: str | None = None + other_software_stack: str | None = None + sw_notes: str | None = None + + @field_validator("system_node_ensemble_id", mode="before") + @classmethod + def _coerce_to_int(cls, v: object) -> object: + if isinstance(v, str): + try: + return int(v) + except ValueError: + return v + return v + + @model_validator(mode="before") + @classmethod + def _lift_flat_accelerator_fields(cls, data: object) -> object: + """Normalize flat accelerator fields into accelerator_info entries.""" + if not isinstance(data, dict) or data.get("accelerator_info"): + return data + flat = { + name: data[name] + for name in _ACCELERATOR_FIELDS + if data.get(name) is not None + } + if flat: + data = {**data, "accelerator_info": [flat]} + return data + + @model_validator(mode="after") + def _require_core_or_vcpu_count(self) -> NodeType: + """A node must disclose at least one of physical core count or vCPU count.""" + if ( + self.host_processor_core_count is None + and self.host_processor_vcpu_count is None + ): + raise ValueError( + "a node type must give host_processor_core_count or host_processor_vcpu_count" + ) + return self + + +class ConfigSummary(BaseModel): + """Structured serving configuration and parallelism disclosure.""" + + model_config = ConfigDict(extra="ignore") + disaggregated: bool | None = None + expert_parallel: int | None = None + tensor_parallel: int | None = None + pipeline_parallel: int | None = None + data_parallel: int | None = None + batch: int | None = None + + +_ConfigSummaryStr = Annotated[str, StringConstraints(min_length=4)] +_AVAILABILITY_SPELLINGS = ( + "publication_status", + "system_availability_status", + "availability_status", +) + + +class DatasetAccuracyScores(BaseModel): + """Per-dataset accuracy scores for ``measured_accuracy_score`` (§8.2).""" + + model_config = ConfigDict(extra="ignore") + scores: dict[str, StrictFloat] + + +class SystemDescription(BaseModel): + """Parsed per-point system description (§8.2).""" + + model_config = ConfigDict(extra="ignore") + submitter_org_names: str | None = None + submitter_contact: str | None = None + submission_id: str | None = None + submission_date: str | None = None + publish_date: str | None = None + cooling: str | None = None + system_name: str + system_category: str | None = None + publication_status: SystemAvailabilityStatus + max_supported_concurrency: int + system_size: str + system_node_ensemble_count: int + system_node_ensemble_total: int + serving_framework: str | None = None + shortened_system_name: str | None = None + endpoint_url: str | None = None + node_types: list[NodeType] + node_config: str | None = None + config_summary: ConfigSummary | _ConfigSummaryStr | None = None + config_summary_notes: str | None = None + disaggregated: bool | None = None + expert_parallel: int | None = None + tensor_parallel: int | None = None + pipeline_parallel: int | None = None + data_parallel: int | None = None + batch: int | None = None + link_config: str | None = None + tps_utilization: float | None = None + division: Division + model_precision: str | None = None + link_to_model: str | None = None + link_to_model_transformation: str | None = None + model_notes: str | None = None + dataset_id: str | None = None + dataset_name: str | None = None + input_token_average: float | None = None + output_token_average: float | None = None + dataset_type: str | None = None + dataset_link: str | None = None + measured_accuracy_score: str | float | dict[str, DatasetAccuracyScores] | None = ( + None + ) + + @field_validator("division", mode="before") + @classmethod + def _coerce_division(cls, v: object) -> object: + if isinstance(v, str): + mapping = { + "standardized": "Standardized", + "serviced": "Serviced", + "rdi": "RDI", + } + normalized = mapping.get(v.strip().lower()) + if normalized is not None: + return normalized + raise ValueError( + f"unknown division {v!r}; must be one of standardized, serviced, rdi" + ) + return v + + @model_validator(mode="before") + @classmethod + def _reconcile_availability_spellings(cls, data: object) -> object: + """Accept availability field aliases and reject conflicting declarations.""" + if not isinstance(data, dict): + return data + present = { + name: data[name] + for name in _AVAILABILITY_SPELLINGS + if data.get(name) not in (None, "") + } + if not present: + return data + distinct = {str(v).strip().lower() for v in present.values()} + if len(distinct) > 1: + pairs = ", ".join((f"{k}={v!r}" for k, v in sorted(present.items()))) + raise ValueError(f"the availability fields disagree ({pairs})") + if data.get("publication_status") in (None, ""): + data = {**data, "publication_status": next(iter(present.values()))} + return data + + @field_validator("publication_status", mode="before") + @classmethod + def _coerce_availability(cls, v: object) -> object: + if isinstance(v, str): + mapping = {"available": "Available", "preview": "Preview", "rdi": "RDI"} + normalized = mapping.get(v.strip().lower()) + if normalized is not None: + return normalized + raise ValueError( + f"unknown availability {v!r}; must be one of available, preview, rdi" + ) + return v + + @field_validator("input_token_average", "output_token_average", mode="before") + @classmethod + def _coerce_tokens_to_float(cls, v: object) -> object: + if isinstance(v, str) and v: + try: + return float(v) + except ValueError: + return v + return v + + @field_validator("measured_accuracy_score", mode="before") + @classmethod + def _coerce_empty_accuracy_to_none(cls, v: object) -> object: + if v == "" or v is None: + return None + return v diff --git a/src/inference_endpoint/validation/results.py b/src/inference_endpoint/validation/results.py new file mode 100644 index 000000000..428258724 --- /dev/null +++ b/src/inference_endpoint/validation/results.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 MLCommons +# SPDX-License-Identifier: Apache-2.0 +"""Structured findings and submission reports.""" + +from __future__ import annotations + +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field, computed_field + +from .types import Severity + + +class CheckResult(BaseModel): + """Result of a single automated check.""" + + model_config = ConfigDict(frozen=True) + rule: str + message: str + severity: Severity = Severity.ERROR + path: Path | None = None + spec_ref: str = "" + key: str = "" + title: str = "" + fix: str | None = None + + @computed_field # type: ignore[misc] + @property + def passed(self) -> bool: + """True when the result is not an error.""" + return self.severity != Severity.ERROR + + +class Report(BaseModel): + """Aggregated results from all checks against a submission.""" + + submission_path: Path + results: list[CheckResult] = Field(default_factory=list) + + @computed_field # type: ignore[misc] + @property + def errors(self) -> list[CheckResult]: + """All results with ERROR severity.""" + return [r for r in self.results if r.severity == Severity.ERROR] + + @computed_field # type: ignore[misc] + @property + def warnings(self) -> list[CheckResult]: + """All results with WARNING severity.""" + return [r for r in self.results if r.severity == Severity.WARNING] + + @computed_field # type: ignore[misc] + @property + def passed(self) -> bool: + """True when there are no errors.""" + return len(self.errors) == 0 diff --git a/tests/unit/validation/test_artifacts.py b/tests/unit/validation/test_artifacts.py new file mode 100644 index 000000000..4770c350f --- /dev/null +++ b/tests/unit/validation/test_artifacts.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Artifact construction, field presence, and input ownership.""" + +import json +from collections import Counter +from pathlib import Path + +import pytest + +from inference_endpoint.config.schema import LoadPatternType +from inference_endpoint.validation import ( + bundled_policy_path, + load_policy, +) +from inference_endpoint.validation.artifacts import PointArtifacts, load_artifacts + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def documents(): + return ( + {"concurrency": 16, "runtime_settings": {"runtime": {}}}, + { + "n_samples_completed": 4, + "duration_ns": 2e9, + "output_sequence_lengths": {"total": 100}, + "system_tps": 999, + }, + {}, + ) + + +def test_point_does_not_retain_input_documents(tmp_path, documents): + config, summary, system = documents + config["checkpoint"] = {"repository": "model", "revision": "a" * 40} + config["unused_payload"] = {"large": [1, 2, 3]} + point = PointArtifacts.from_json(tmp_path / "model" / "r16", *documents) + config["checkpoint"]["revision"] = "b" * 40 + summary["output_sequence_lengths"]["total"] = 10000 + system["system_name"] = "changed" + assert point.config.checkpoint.revision == "a" * 40 + assert point.throughput == 50 + assert not point.config.model_extra + assert not hasattr(point.config, "unused_payload") + assert not any( + hasattr(point, name) for name in ("raw", "summary_raw", "system_raw") + ) + + +def test_supplied_fields_distinguish_omissions_from_defaults(tmp_path, documents): + config, _, _ = documents + config["runtime_settings"]["runtime"]["scheduler_random_seed"] = 123 + point = PointArtifacts.from_json(tmp_path, *documents) + assert point.config.runtime_settings.load_pattern is LoadPatternType.CONCURRENCY + assert point.context.load_pattern is None + assert point.get_config("runtime_settings.stream_all_chunks") is None + assert point.config.runtime_settings.runtime.scheduler_rng_seed == 123 + assert ( + "runtime_settings.runtime.scheduler_rng_seed" + not in point.evidence.config.fields + ) + assert ( + "runtime_settings.runtime.scheduler_random_seed" in point.evidence.config.fields + ) + config["runtime_settings"].update( + load_pattern="concurrency", stream_all_chunks=False + ) + explicit = PointArtifacts.from_json(tmp_path, *documents) + assert explicit.context.load_pattern is LoadPatternType.CONCURRENCY + assert explicit.get_config("runtime_settings.stream_all_chunks") is False + + +def test_reported_metrics_do_not_resolve_to_calculated_values(tmp_path, documents): + point = PointArtifacts.from_json(tmp_path, *documents) + assert point.summary.system_tps == 50 + assert point.get_summary("system_tps") == 999 + del documents[1]["system_tps"] + omitted = PointArtifacts.from_json(tmp_path, *documents) + assert omitted.summary.system_tps == 50 + assert omitted.get_summary("system_tps") is None + + +@pytest.mark.parametrize("standalone_accuracy", [False, True]) +def test_point_input_files_are_read_once( + tmp_path, documents, monkeypatch, standalone_accuracy +): + point = tmp_path / "results" / "system" / "gpt-oss-120b" / "r16" + point.mkdir(parents=True) + for name, data in zip( + ("point.yaml", "result_summary.json", "system_desc.json"), + documents, + strict=True, + ): + (point / name).write_text(json.dumps(data)) + accuracy = {"aime25": {"num_samples": 30, "score": 90}} + accuracy_name = "accuracy_results.json" if standalone_accuracy else "results.json" + (point / accuracy_name).write_text( + json.dumps(accuracy if standalone_accuracy else {"accuracy_scores": accuracy}) + ) + reads = Counter() + original = Path.read_text + + def read_text(path, *args, **kwargs): + reads[path] += 1 + return original(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read_text) + artifacts = load_artifacts(tmp_path, load_policy(bundled_policy_path())) + assert ( + artifacts.index.points[str(point)].accuracy.root["aime25"]["num_samples"] == 30 + ) + assert { + path.name: count for path, count in reads.items() if path.parent == point + } == { + "point.yaml": 1, + "result_summary.json": 1, + "system_desc.json": 1, + accuracy_name: 1, + } + + +@pytest.mark.parametrize("field", ["duration_ns", "output_sequence_lengths"]) +def test_throughput_requires_supplied_operands(tmp_path, documents, field): + del documents[1][field] + point = PointArtifacts.from_json(tmp_path, *documents) + assert point.throughput is None diff --git a/tests/unit/validation/test_evidence.py b/tests/unit/validation/test_evidence.py new file mode 100644 index 000000000..7a2249e1b --- /dev/null +++ b/tests/unit/validation/test_evidence.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Structural evidence parsing and policy/evidence separation.""" + +import json + +import pytest +from pydantic import ValidationError + +from inference_endpoint.validation.catalogs.seeds import ( + SeedSetError, + parse_seed_catalog, +) +from inference_endpoint.validation.evidence.accuracy import AccuracyResult +from inference_endpoint.validation.evidence.loaders import ( + load_point_config, + load_result_summary, +) +from inference_endpoint.validation.power.models import SourcedValue + +pytestmark = pytest.mark.unit + + +def test_point_parsing_does_not_enforce_submission_policy(tmp_path): + path = tmp_path / "point.yaml" + path.write_text("concurrency: 1\nruntime_settings:\n runtime: {}\n") + model, findings = load_point_config(path) + assert model is not None + assert not findings + assert not hasattr(model, "_check_results") + + +def test_native_accuracy_aliases_and_duplicate_datasets(): + entry = { + "dataset_name": "aime25", + "score": 90, + "unit_samples": 30, + "num_repeats": 2, + } + model = AccuracyResult.model_validate({"accuracy_scores": [entry]}) + assert model.root["aime25"]["num_samples"] == 30 + assert model.root["aime25"]["n_repeats"] == 2 + with pytest.raises(ValidationError, match="appears more than once"): + AccuracyResult.model_validate({"accuracy_scores": [entry, entry]}) + + +def test_unreadable_encoding_becomes_structured_schema_error(tmp_path): + path = tmp_path / "result_summary.json" + path.write_bytes(b"\xff") + model, findings = load_result_summary(path) + assert model is None + assert findings[0].key == "artifact-unreadable" + + +@pytest.mark.parametrize("value", [float("inf"), float("nan")]) +def test_power_source_requires_finite_value(value): + with pytest.raises(ValidationError): + SourcedValue.model_validate( + { + "value_w": value, + "source_type": "vendor_spec", + "source": "https://example.com/power", + } + ) + + +@pytest.mark.parametrize("value", [True, 1.5, "1"]) +def test_seed_catalog_rejects_lossy_integer_coercion(tmp_path, value): + path = tmp_path / "seed_sets.yaml" + path.write_text( + json.dumps( + { + "seed_sets": [ + { + "id": "A", + "scheduler_rng_seed": value, + "sample_index_rng_seed": 2, + "model_seed": 3, + } + ] + } + ) + ) + with pytest.raises(SeedSetError, match="must define integer"): + parse_seed_catalog(path) + + +def test_seed_catalog_rejects_duplicate_ids(tmp_path): + path = tmp_path / "seed_sets.yaml" + entry = { + "id": "A", + "scheduler_rng_seed": 1, + "sample_index_rng_seed": 2, + "model_seed": 3, + } + path.write_text(json.dumps({"seed_sets": [entry, entry]})) + with pytest.raises(SeedSetError, match="duplicate"): + parse_seed_catalog(path) diff --git a/tests/unit/validation/test_policy_contracts.py b/tests/unit/validation/test_policy_contracts.py index 319b9bf6f..e776bf834 100644 --- a/tests/unit/validation/test_policy_contracts.py +++ b/tests/unit/validation/test_policy_contracts.py @@ -6,12 +6,52 @@ from pydantic import ValidationError from inference_endpoint.validation import bundled_policy_path, load_policy +from inference_endpoint.validation.artifacts import PointArtifacts +from inference_endpoint.validation.evidence.accuracy import AccuracyResult +from inference_endpoint.validation.planner import plan_checks from inference_endpoint.validation.power.calculation import PowerCalculator from inference_endpoint.validation.power.models import SystemPower +from inference_endpoint.validation.types import Decision, EvidenceKey pytestmark = pytest.mark.unit +def test_accuracy_nested_scores_are_owned(): + data = {"aime25": {"score": {"exact_match": 90}, "extras": {"source": "test"}}} + parsed = AccuracyResult.model_validate(data) + data["aime25"]["score"]["exact_match"] = 0 + data["aime25"]["extras"]["source"] = "changed" + assert parsed.metric_scores() == {"aime25": {"exact_match": 90.0}} + assert parsed.root["aime25"]["extras"]["source"] == "test" + + +@pytest.mark.parametrize("field", ["speculative_decoding", "drafter"]) +@pytest.mark.parametrize("declaration", [{}, {"target_checksum": "declared-checksum"}]) +def test_supplied_decode_head_keeps_approval_checks_selected( + tmp_path, field, declaration +): + point = PointArtifacts.from_json( + tmp_path / "kimi-k3" / "r16", + { + "concurrency": 16, + "runtime_settings": {"runtime": {}, "load_pattern": "agentic_inference"}, + field: declaration, + }, + {"n_samples_completed": 1, "duration_ns": 1e9}, + {}, + ) + point = PointArtifacts.from_evidence( + point.path, point.evidence, frozenset({EvidenceKey.APPROVED_SPED_DECODE_HEADS}) + ) + policy = load_policy(bundled_policy_path()) + checks = { + check.rule.id: check for check in plan_checks(policy, [point.context]).checks + } + assert point.context.speculative_decoding is True + assert checks["approved-drafter"].decision is Decision.READY + assert checks["drafter-approval-lead-time"].decision is Decision.READY + + @pytest.fixture def policy_dir(tmp_path): directory = tmp_path / "2026-10-C1" @@ -85,6 +125,41 @@ def test_unknown_cooling_stays_invalid(): SystemPower.model_validate({"cooling": "unknown"}) +def test_finite_numeric_strings_in_accuracy_are_normalized(): + parsed = AccuracyResult.model_validate( + { + "rouge": {"score": {"rouge1": "45.12", "rouge2": 22.01}}, + "scalar": {"score": "83.13"}, + } + ) + assert parsed.metric_scores() == { + "rouge": {"rouge1": 45.12, "rouge2": 22.01}, + "scalar": {"score": 83.13}, + } + assert parsed.root["rouge"]["score"]["rouge1"] == 45.12 + + +@pytest.mark.parametrize("declaration", [None, "absent"]) +def test_no_decode_head_keeps_approval_checks_excluded(tmp_path, declaration): + config = {"concurrency": 16, "runtime_settings": {"runtime": {}}} + if declaration is None: + config["speculative_decoding"] = None + point = PointArtifacts.from_json( + tmp_path / "kimi-k3" / "r16", + config, + {"n_samples_completed": 1, "duration_ns": 1e9}, + {}, + ) + checks = { + check.rule.id: check + for check in plan_checks( + load_policy(bundled_policy_path()), [point.context] + ).checks + } + assert point.context.speculative_decoding is False + assert checks["approved-drafter"].decision is Decision.EXCLUDED + + def test_valid_override_retains_false_and_numeric_values(policy_dir): path = policy_dir / "catalog.yaml" document = yaml.safe_load(path.read_text()) diff --git a/tests/unit/validation/test_runtime_safety.py b/tests/unit/validation/test_runtime_safety.py new file mode 100644 index 000000000..8481ff818 --- /dev/null +++ b/tests/unit/validation/test_runtime_safety.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Runtime safety regressions for malformed evidence and repeated validation.""" + +import json + +import pytest + +from inference_endpoint.validation.evidence.loaders import load_result_summary +from inference_endpoint.validation.types import Severity + +pytestmark = pytest.mark.unit + + +@pytest.mark.parametrize( + "field", ["tps_per_user", "system_tps", "e2e_avg_interactivity"] +) +@pytest.mark.parametrize( + "value", [[], {}, "NaN", "Infinity", float("nan"), float("inf")] +) +def test_invalid_stored_metrics_are_structured_load_errors(tmp_path, field, value): + path = tmp_path / "results_summary.json" + path.write_text( + json.dumps({"n_samples_completed": 1, "duration_ns": 1e9, field: value}) + ) + model, results = load_result_summary(path) + assert model is None + assert results + assert all(result.severity == Severity.ERROR for result in results) From 920d57f155f66096a23cc9a243a820908b5ff1d0 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 01:34:01 -0700 Subject: [PATCH 06/18] feat(validation): execute submission checks with callable evaluators --- .gitignore | 2 + src/inference_endpoint/validation/__init__.py | 5 + src/inference_endpoint/validation/api.py | 44 ++ src/inference_endpoint/validation/bindings.py | 125 +++++ .../validation/checks/__init__.py | 7 + .../validation/checks/accuracy.py | 470 +++++++++++++++++ .../validation/checks/collections.py | 265 ++++++++++ .../validation/checks/datasets.py | 110 ++++ .../validation/checks/drafters.py | 95 ++++ .../validation/checks/helpers.py | 111 ++++ .../validation/checks/metrics.py | 227 +++++++++ .../validation/checks/power.py | 450 +++++++++++++++++ .../validation/checks/seeds.py | 144 ++++++ .../validation/checks/steady_state.py | 136 +++++ .../validation/checks/structure.py | 269 ++++++++++ .../validation/checks/warmup.py | 85 ++++ src/inference_endpoint/validation/engine.py | 81 +++ .../validation/evaluator_base.py | 30 ++ src/inference_endpoint/validation/outcomes.py | 32 ++ .../invalid_submission/docs/.gitkeep | 0 .../r16/accuracy_results.json | 13 + .../Llama-3_1-8B-Instruct/r16/config.yaml | 1 + .../Llama-3_1-8B-Instruct/r16/point.yaml | 49 ++ .../r16/result_summary.json | 29 ++ .../r16/system_desc.json | 60 +++ .../r38/accuracy_results.json | 13 + .../Llama-3_1-8B-Instruct/r38/config.yaml | 1 + .../Llama-3_1-8B-Instruct/r38/point.yaml | 49 ++ .../r38/result_summary.json | 29 ++ .../r38/system_desc.json | 60 +++ .../r88/accuracy_results.json | 13 + .../Llama-3_1-8B-Instruct/r88/config.yaml | 1 + .../Llama-3_1-8B-Instruct/r88/point.yaml | 50 ++ .../r88/result_summary.json | 29 ++ .../r88/system_desc.json | 60 +++ .../results/bad_system/system_power.json | 44 ++ .../invalid_submission/src/custom/README.md | 3 + .../submissions/sub_a/docs/.gitkeep | 0 .../gpt-oss-120b/r1024/accuracy_results.json | 11 + .../gpt-oss-120b/r1024/config.yaml | 1 + .../gpt-oss-120b/r1024/point.yaml | 49 ++ .../gpt-oss-120b/r1024/result_summary.json | 29 ++ .../gpt-oss-120b/r1024/system_desc.json | 60 +++ .../gpt-oss-120b/r128/accuracy_results.json | 11 + .../gpt-oss-120b/r128/config.yaml | 1 + .../gpt-oss-120b/r128/point.yaml | 49 ++ .../gpt-oss-120b/r128/result_summary.json | 29 ++ .../gpt-oss-120b/r128/system_desc.json | 60 +++ .../gpt-oss-120b/r16/accuracy_results.json | 11 + .../gpt-oss-120b/r16/config.yaml | 1 + .../gpt-oss-120b/r16/point.yaml | 49 ++ .../gpt-oss-120b/r16/result_summary.json | 29 ++ .../gpt-oss-120b/r16/system_desc.json | 60 +++ .../gpt-oss-120b/r2048/accuracy_results.json | 11 + .../gpt-oss-120b/r2048/config.yaml | 1 + .../gpt-oss-120b/r2048/point.yaml | 50 ++ .../gpt-oss-120b/r2048/result_summary.json | 29 ++ .../gpt-oss-120b/r2048/system_desc.json | 60 +++ .../gpt-oss-120b/r4/accuracy_results.json | 11 + .../gpt-oss-120b/r4/config.yaml | 1 + .../gpt-oss-120b/r4/point.yaml | 49 ++ .../gpt-oss-120b/r4/result_summary.json | 29 ++ .../gpt-oss-120b/r4/system_desc.json | 60 +++ .../gpt-oss-120b/r512/accuracy_results.json | 11 + .../gpt-oss-120b/r512/config.yaml | 1 + .../gpt-oss-120b/r512/point.yaml | 49 ++ .../gpt-oss-120b/r512/result_summary.json | 29 ++ .../gpt-oss-120b/r512/system_desc.json | 60 +++ .../gpt-oss-120b/r64/accuracy_results.json | 11 + .../gpt-oss-120b/r64/config.yaml | 1 + .../gpt-oss-120b/r64/point.yaml | 49 ++ .../gpt-oss-120b/r64/result_summary.json | 29 ++ .../gpt-oss-120b/r64/system_desc.json | 60 +++ .../results/sys_mi355x_8gpu/system_power.json | 44 ++ .../submissions/sub_a/src/vllm/README.md | 3 + .../submissions/sub_b/docs/.gitkeep | 0 .../gpt-oss-120b/r1024/accuracy_results.json | 11 + .../gpt-oss-120b/r1024/config.yaml | 1 + .../gpt-oss-120b/r1024/point.yaml | 49 ++ .../gpt-oss-120b/r1024/result_summary.json | 29 ++ .../gpt-oss-120b/r1024/system_desc.json | 60 +++ .../gpt-oss-120b/r128/accuracy_results.json | 11 + .../gpt-oss-120b/r128/config.yaml | 1 + .../gpt-oss-120b/r128/point.yaml | 49 ++ .../gpt-oss-120b/r128/result_summary.json | 29 ++ .../gpt-oss-120b/r128/system_desc.json | 60 +++ .../gpt-oss-120b/r16/accuracy_results.json | 11 + .../gpt-oss-120b/r16/config.yaml | 1 + .../gpt-oss-120b/r16/point.yaml | 49 ++ .../gpt-oss-120b/r16/result_summary.json | 29 ++ .../gpt-oss-120b/r16/system_desc.json | 60 +++ .../gpt-oss-120b/r2048/accuracy_results.json | 11 + .../gpt-oss-120b/r2048/config.yaml | 1 + .../gpt-oss-120b/r2048/point.yaml | 50 ++ .../gpt-oss-120b/r2048/result_summary.json | 29 ++ .../gpt-oss-120b/r2048/system_desc.json | 60 +++ .../gpt-oss-120b/r4/accuracy_results.json | 11 + .../gpt-oss-120b/r4/config.yaml | 1 + .../gpt-oss-120b/r4/point.yaml | 49 ++ .../gpt-oss-120b/r4/result_summary.json | 29 ++ .../gpt-oss-120b/r4/system_desc.json | 60 +++ .../gpt-oss-120b/r512/accuracy_results.json | 11 + .../gpt-oss-120b/r512/config.yaml | 1 + .../gpt-oss-120b/r512/point.yaml | 49 ++ .../gpt-oss-120b/r512/result_summary.json | 29 ++ .../gpt-oss-120b/r512/system_desc.json | 60 +++ .../gpt-oss-120b/r64/accuracy_results.json | 11 + .../gpt-oss-120b/r64/config.yaml | 1 + .../gpt-oss-120b/r64/point.yaml | 49 ++ .../gpt-oss-120b/r64/result_summary.json | 29 ++ .../gpt-oss-120b/r64/system_desc.json | 60 +++ .../sys_mi355x_16gpu/system_power.json | 44 ++ .../submissions/sub_b/src/vllm/README.md | 3 + .../submissions/sub_c/docs/.gitkeep | 0 .../r128/accuracy_results.json | 9 + .../qwen3-coder-480b/r128/config.yaml | 1 + .../qwen3-coder-480b/r128/point.yaml | 49 ++ .../qwen3-coder-480b/r128/result_summary.json | 29 ++ .../qwen3-coder-480b/r128/system_desc.json | 60 +++ .../r16/accuracy_results.json | 9 + .../qwen3-coder-480b/r16/config.yaml | 1 + .../qwen3-coder-480b/r16/point.yaml | 49 ++ .../qwen3-coder-480b/r16/result_summary.json | 29 ++ .../qwen3-coder-480b/r16/system_desc.json | 60 +++ .../r256/accuracy_results.json | 9 + .../qwen3-coder-480b/r256/config.yaml | 1 + .../qwen3-coder-480b/r256/point.yaml | 49 ++ .../qwen3-coder-480b/r256/result_summary.json | 29 ++ .../qwen3-coder-480b/r256/system_desc.json | 60 +++ .../r32/accuracy_results.json | 9 + .../qwen3-coder-480b/r32/config.yaml | 1 + .../qwen3-coder-480b/r32/point.yaml | 49 ++ .../qwen3-coder-480b/r32/result_summary.json | 29 ++ .../qwen3-coder-480b/r32/system_desc.json | 60 +++ .../r512/accuracy_results.json | 9 + .../qwen3-coder-480b/r512/config.yaml | 1 + .../qwen3-coder-480b/r512/point.yaml | 50 ++ .../qwen3-coder-480b/r512/result_summary.json | 29 ++ .../qwen3-coder-480b/r512/system_desc.json | 60 +++ .../r64/accuracy_results.json | 9 + .../qwen3-coder-480b/r64/config.yaml | 1 + .../qwen3-coder-480b/r64/point.yaml | 49 ++ .../qwen3-coder-480b/r64/result_summary.json | 29 ++ .../qwen3-coder-480b/r64/system_desc.json | 60 +++ .../qwen3-coder-480b/r8/accuracy_results.json | 9 + .../qwen3-coder-480b/r8/config.yaml | 1 + .../qwen3-coder-480b/r8/point.yaml | 49 ++ .../qwen3-coder-480b/r8/result_summary.json | 29 ++ .../qwen3-coder-480b/r8/system_desc.json | 60 +++ .../results/sys_tpu_4chip/system_power.json | 44 ++ .../submissions/sub_c/src/vllm/README.md | 3 + .../submissions/sub_d/docs/.gitkeep | 0 .../r1024/accuracy_results.json | 9 + .../qwen3-coder-480b/r1024/config.yaml | 1 + .../qwen3-coder-480b/r1024/point.yaml | 50 ++ .../r1024/result_summary.json | 29 ++ .../qwen3-coder-480b/r1024/system_desc.json | 60 +++ .../r128/accuracy_results.json | 9 + .../qwen3-coder-480b/r128/config.yaml | 1 + .../qwen3-coder-480b/r128/point.yaml | 49 ++ .../qwen3-coder-480b/r128/result_summary.json | 29 ++ .../qwen3-coder-480b/r128/system_desc.json | 60 +++ .../r16/accuracy_results.json | 9 + .../qwen3-coder-480b/r16/config.yaml | 1 + .../qwen3-coder-480b/r16/point.yaml | 49 ++ .../qwen3-coder-480b/r16/result_summary.json | 29 ++ .../qwen3-coder-480b/r16/system_desc.json | 60 +++ .../r256/accuracy_results.json | 9 + .../qwen3-coder-480b/r256/config.yaml | 1 + .../qwen3-coder-480b/r256/point.yaml | 49 ++ .../qwen3-coder-480b/r256/result_summary.json | 29 ++ .../qwen3-coder-480b/r256/system_desc.json | 60 +++ .../r32/accuracy_results.json | 9 + .../qwen3-coder-480b/r32/config.yaml | 1 + .../qwen3-coder-480b/r32/point.yaml | 49 ++ .../qwen3-coder-480b/r32/result_summary.json | 29 ++ .../qwen3-coder-480b/r32/system_desc.json | 60 +++ .../r512/accuracy_results.json | 9 + .../qwen3-coder-480b/r512/config.yaml | 1 + .../qwen3-coder-480b/r512/point.yaml | 49 ++ .../qwen3-coder-480b/r512/result_summary.json | 29 ++ .../qwen3-coder-480b/r512/system_desc.json | 60 +++ .../r64/accuracy_results.json | 9 + .../qwen3-coder-480b/r64/config.yaml | 1 + .../qwen3-coder-480b/r64/point.yaml | 49 ++ .../qwen3-coder-480b/r64/result_summary.json | 29 ++ .../qwen3-coder-480b/r64/system_desc.json | 60 +++ .../qwen3-coder-480b/r8/accuracy_results.json | 9 + .../qwen3-coder-480b/r8/config.yaml | 1 + .../qwen3-coder-480b/r8/point.yaml | 49 ++ .../qwen3-coder-480b/r8/result_summary.json | 29 ++ .../qwen3-coder-480b/r8/system_desc.json | 60 +++ .../results/sys_tpu_8chip/system_power.json | 44 ++ .../submissions/sub_d/src/vllm/README.md | 3 + .../submissions/sub_e/docs/.gitkeep | 0 .../llama3-8b/r1/accuracy_results.json | 15 + .../sys_gaudi_dp1/llama3-8b/r1/config.yaml | 1 + .../sys_gaudi_dp1/llama3-8b/r1/point.yaml | 49 ++ .../llama3-8b/r1/result_summary.json | 29 ++ .../llama3-8b/r1/system_desc.json | 60 +++ .../llama3-8b/r1024/accuracy_results.json | 15 + .../sys_gaudi_dp1/llama3-8b/r1024/config.yaml | 1 + .../sys_gaudi_dp1/llama3-8b/r1024/point.yaml | 50 ++ .../llama3-8b/r1024/result_summary.json | 29 ++ .../llama3-8b/r1024/system_desc.json | 60 +++ .../llama3-8b/r128/accuracy_results.json | 15 + .../sys_gaudi_dp1/llama3-8b/r128/config.yaml | 1 + .../sys_gaudi_dp1/llama3-8b/r128/point.yaml | 49 ++ .../llama3-8b/r128/result_summary.json | 29 ++ .../llama3-8b/r128/system_desc.json | 60 +++ .../llama3-8b/r16/accuracy_results.json | 15 + .../sys_gaudi_dp1/llama3-8b/r16/config.yaml | 1 + .../sys_gaudi_dp1/llama3-8b/r16/point.yaml | 49 ++ .../llama3-8b/r16/result_summary.json | 29 ++ .../llama3-8b/r16/system_desc.json | 60 +++ .../llama3-8b/r2/accuracy_results.json | 15 + .../sys_gaudi_dp1/llama3-8b/r2/config.yaml | 1 + .../sys_gaudi_dp1/llama3-8b/r2/point.yaml | 49 ++ .../llama3-8b/r2/result_summary.json | 29 ++ .../llama3-8b/r2/system_desc.json | 60 +++ .../llama3-8b/r256/accuracy_results.json | 15 + .../sys_gaudi_dp1/llama3-8b/r256/config.yaml | 1 + .../sys_gaudi_dp1/llama3-8b/r256/point.yaml | 49 ++ .../llama3-8b/r256/result_summary.json | 29 ++ .../llama3-8b/r256/system_desc.json | 60 +++ .../llama3-8b/r32/accuracy_results.json | 15 + .../sys_gaudi_dp1/llama3-8b/r32/config.yaml | 1 + .../sys_gaudi_dp1/llama3-8b/r32/point.yaml | 49 ++ .../llama3-8b/r32/result_summary.json | 29 ++ .../llama3-8b/r32/system_desc.json | 60 +++ .../llama3-8b/r4/accuracy_results.json | 15 + .../sys_gaudi_dp1/llama3-8b/r4/config.yaml | 1 + .../sys_gaudi_dp1/llama3-8b/r4/point.yaml | 49 ++ .../llama3-8b/r4/result_summary.json | 29 ++ .../llama3-8b/r4/system_desc.json | 60 +++ .../llama3-8b/r512/accuracy_results.json | 15 + .../sys_gaudi_dp1/llama3-8b/r512/config.yaml | 1 + .../sys_gaudi_dp1/llama3-8b/r512/point.yaml | 49 ++ .../llama3-8b/r512/result_summary.json | 29 ++ .../llama3-8b/r512/system_desc.json | 60 +++ .../llama3-8b/r64/accuracy_results.json | 15 + .../sys_gaudi_dp1/llama3-8b/r64/config.yaml | 1 + .../sys_gaudi_dp1/llama3-8b/r64/point.yaml | 49 ++ .../llama3-8b/r64/result_summary.json | 29 ++ .../llama3-8b/r64/system_desc.json | 60 +++ .../llama3-8b/r8/accuracy_results.json | 15 + .../sys_gaudi_dp1/llama3-8b/r8/config.yaml | 1 + .../sys_gaudi_dp1/llama3-8b/r8/point.yaml | 49 ++ .../llama3-8b/r8/result_summary.json | 29 ++ .../llama3-8b/r8/system_desc.json | 60 +++ .../results/sys_gaudi_dp1/system_power.json | 44 ++ .../submissions/sub_e/src/vllm/README.md | 3 + .../submissions/sub_f/docs/.gitkeep | 0 .../llama3-8b/r1/accuracy_results.json | 15 + .../sys_gaudi_dp2/llama3-8b/r1/config.yaml | 1 + .../sys_gaudi_dp2/llama3-8b/r1/point.yaml | 49 ++ .../llama3-8b/r1/result_summary.json | 29 ++ .../llama3-8b/r1/system_desc.json | 60 +++ .../llama3-8b/r1024/accuracy_results.json | 15 + .../sys_gaudi_dp2/llama3-8b/r1024/config.yaml | 1 + .../sys_gaudi_dp2/llama3-8b/r1024/point.yaml | 50 ++ .../llama3-8b/r1024/result_summary.json | 29 ++ .../llama3-8b/r1024/system_desc.json | 60 +++ .../llama3-8b/r128/accuracy_results.json | 15 + .../sys_gaudi_dp2/llama3-8b/r128/config.yaml | 1 + .../sys_gaudi_dp2/llama3-8b/r128/point.yaml | 49 ++ .../llama3-8b/r128/result_summary.json | 29 ++ .../llama3-8b/r128/system_desc.json | 60 +++ .../llama3-8b/r16/accuracy_results.json | 15 + .../sys_gaudi_dp2/llama3-8b/r16/config.yaml | 1 + .../sys_gaudi_dp2/llama3-8b/r16/point.yaml | 49 ++ .../llama3-8b/r16/result_summary.json | 29 ++ .../llama3-8b/r16/system_desc.json | 60 +++ .../llama3-8b/r2/accuracy_results.json | 15 + .../sys_gaudi_dp2/llama3-8b/r2/config.yaml | 1 + .../sys_gaudi_dp2/llama3-8b/r2/point.yaml | 49 ++ .../llama3-8b/r2/result_summary.json | 29 ++ .../llama3-8b/r2/system_desc.json | 60 +++ .../llama3-8b/r256/accuracy_results.json | 15 + .../sys_gaudi_dp2/llama3-8b/r256/config.yaml | 1 + .../sys_gaudi_dp2/llama3-8b/r256/point.yaml | 49 ++ .../llama3-8b/r256/result_summary.json | 29 ++ .../llama3-8b/r256/system_desc.json | 60 +++ .../llama3-8b/r32/accuracy_results.json | 15 + .../sys_gaudi_dp2/llama3-8b/r32/config.yaml | 1 + .../sys_gaudi_dp2/llama3-8b/r32/point.yaml | 49 ++ .../llama3-8b/r32/result_summary.json | 29 ++ .../llama3-8b/r32/system_desc.json | 60 +++ .../llama3-8b/r4/accuracy_results.json | 15 + .../sys_gaudi_dp2/llama3-8b/r4/config.yaml | 1 + .../sys_gaudi_dp2/llama3-8b/r4/point.yaml | 49 ++ .../llama3-8b/r4/result_summary.json | 29 ++ .../llama3-8b/r4/system_desc.json | 60 +++ .../llama3-8b/r512/accuracy_results.json | 15 + .../sys_gaudi_dp2/llama3-8b/r512/config.yaml | 1 + .../sys_gaudi_dp2/llama3-8b/r512/point.yaml | 49 ++ .../llama3-8b/r512/result_summary.json | 29 ++ .../llama3-8b/r512/system_desc.json | 60 +++ .../llama3-8b/r64/accuracy_results.json | 15 + .../sys_gaudi_dp2/llama3-8b/r64/config.yaml | 1 + .../sys_gaudi_dp2/llama3-8b/r64/point.yaml | 49 ++ .../llama3-8b/r64/result_summary.json | 29 ++ .../llama3-8b/r64/system_desc.json | 60 +++ .../llama3-8b/r8/accuracy_results.json | 15 + .../sys_gaudi_dp2/llama3-8b/r8/config.yaml | 1 + .../sys_gaudi_dp2/llama3-8b/r8/point.yaml | 49 ++ .../llama3-8b/r8/result_summary.json | 29 ++ .../llama3-8b/r8/system_desc.json | 60 +++ .../results/sys_gaudi_dp2/system_power.json | 68 +++ .../submissions/sub_f/src/vllm/README.md | 3 + .../submissions/sub_g/docs/.gitkeep | 0 .../llama3-70b/r1024/accuracy_results.json | 9 + .../llama3-70b/r1024/config.yaml | 1 + .../sys_8gpu_vllm/llama3-70b/r1024/point.yaml | 49 ++ .../llama3-70b/r1024/result_summary.json | 29 ++ .../llama3-70b/r1024/system_desc.json | 61 +++ .../llama3-70b/r128/accuracy_results.json | 9 + .../sys_8gpu_vllm/llama3-70b/r128/config.yaml | 1 + .../sys_8gpu_vllm/llama3-70b/r128/point.yaml | 49 ++ .../llama3-70b/r128/result_summary.json | 29 ++ .../llama3-70b/r128/system_desc.json | 61 +++ .../llama3-70b/r1536/accuracy_results.json | 9 + .../llama3-70b/r1536/config.yaml | 1 + .../sys_8gpu_vllm/llama3-70b/r1536/point.yaml | 49 ++ .../llama3-70b/r1536/result_summary.json | 29 ++ .../llama3-70b/r1536/system_desc.json | 61 +++ .../llama3-70b/r192/accuracy_results.json | 9 + .../sys_8gpu_vllm/llama3-70b/r192/config.yaml | 1 + .../sys_8gpu_vllm/llama3-70b/r192/point.yaml | 49 ++ .../llama3-70b/r192/result_summary.json | 29 ++ .../llama3-70b/r192/system_desc.json | 61 +++ .../llama3-70b/r2048/accuracy_results.json | 9 + .../llama3-70b/r2048/config.yaml | 1 + .../sys_8gpu_vllm/llama3-70b/r2048/point.yaml | 50 ++ .../llama3-70b/r2048/result_summary.json | 29 ++ .../llama3-70b/r2048/system_desc.json | 61 +++ .../llama3-70b/r256/accuracy_results.json | 9 + .../sys_8gpu_vllm/llama3-70b/r256/config.yaml | 1 + .../sys_8gpu_vllm/llama3-70b/r256/point.yaml | 49 ++ .../llama3-70b/r256/result_summary.json | 29 ++ .../llama3-70b/r256/system_desc.json | 61 +++ .../llama3-70b/r384/accuracy_results.json | 9 + .../sys_8gpu_vllm/llama3-70b/r384/config.yaml | 1 + .../sys_8gpu_vllm/llama3-70b/r384/point.yaml | 49 ++ .../llama3-70b/r384/result_summary.json | 29 ++ .../llama3-70b/r384/system_desc.json | 61 +++ .../llama3-70b/r512/accuracy_results.json | 9 + .../sys_8gpu_vllm/llama3-70b/r512/config.yaml | 1 + .../sys_8gpu_vllm/llama3-70b/r512/point.yaml | 49 ++ .../llama3-70b/r512/result_summary.json | 29 ++ .../llama3-70b/r512/system_desc.json | 61 +++ .../llama3-70b/r64/accuracy_results.json | 9 + .../sys_8gpu_vllm/llama3-70b/r64/config.yaml | 1 + .../sys_8gpu_vllm/llama3-70b/r64/point.yaml | 49 ++ .../llama3-70b/r64/result_summary.json | 29 ++ .../llama3-70b/r64/system_desc.json | 61 +++ .../llama3-70b/r768/accuracy_results.json | 9 + .../sys_8gpu_vllm/llama3-70b/r768/config.yaml | 1 + .../sys_8gpu_vllm/llama3-70b/r768/point.yaml | 49 ++ .../llama3-70b/r768/result_summary.json | 29 ++ .../llama3-70b/r768/system_desc.json | 61 +++ .../results/sys_8gpu_vllm/system_power.json | 44 ++ .../submissions/sub_g/src/vllm/README.md | 3 + .../submissions/sub_h/docs/.gitkeep | 0 .../llama3-70b/r1024/accuracy_results.json | 9 + .../llama3-70b/r1024/config.yaml | 1 + .../llama3-70b/r1024/point.yaml | 49 ++ .../llama3-70b/r1024/result_summary.json | 29 ++ .../llama3-70b/r1024/system_desc.json | 61 +++ .../llama3-70b/r128/accuracy_results.json | 9 + .../llama3-70b/r128/config.yaml | 1 + .../llama3-70b/r128/point.yaml | 49 ++ .../llama3-70b/r128/result_summary.json | 29 ++ .../llama3-70b/r128/system_desc.json | 61 +++ .../llama3-70b/r1536/accuracy_results.json | 9 + .../llama3-70b/r1536/config.yaml | 1 + .../llama3-70b/r1536/point.yaml | 49 ++ .../llama3-70b/r1536/result_summary.json | 29 ++ .../llama3-70b/r1536/system_desc.json | 61 +++ .../llama3-70b/r192/accuracy_results.json | 9 + .../llama3-70b/r192/config.yaml | 1 + .../llama3-70b/r192/point.yaml | 49 ++ .../llama3-70b/r192/result_summary.json | 29 ++ .../llama3-70b/r192/system_desc.json | 61 +++ .../llama3-70b/r2048/accuracy_results.json | 9 + .../llama3-70b/r2048/config.yaml | 1 + .../llama3-70b/r2048/point.yaml | 50 ++ .../llama3-70b/r2048/result_summary.json | 29 ++ .../llama3-70b/r2048/system_desc.json | 61 +++ .../llama3-70b/r256/accuracy_results.json | 9 + .../llama3-70b/r256/config.yaml | 1 + .../llama3-70b/r256/point.yaml | 49 ++ .../llama3-70b/r256/result_summary.json | 29 ++ .../llama3-70b/r256/system_desc.json | 61 +++ .../llama3-70b/r384/accuracy_results.json | 9 + .../llama3-70b/r384/config.yaml | 1 + .../llama3-70b/r384/point.yaml | 49 ++ .../llama3-70b/r384/result_summary.json | 29 ++ .../llama3-70b/r384/system_desc.json | 61 +++ .../llama3-70b/r512/accuracy_results.json | 9 + .../llama3-70b/r512/config.yaml | 1 + .../llama3-70b/r512/point.yaml | 49 ++ .../llama3-70b/r512/result_summary.json | 29 ++ .../llama3-70b/r512/system_desc.json | 61 +++ .../llama3-70b/r64/accuracy_results.json | 9 + .../llama3-70b/r64/config.yaml | 1 + .../sys_8gpu_sglang/llama3-70b/r64/point.yaml | 49 ++ .../llama3-70b/r64/result_summary.json | 29 ++ .../llama3-70b/r64/system_desc.json | 61 +++ .../llama3-70b/r768/accuracy_results.json | 9 + .../llama3-70b/r768/config.yaml | 1 + .../llama3-70b/r768/point.yaml | 49 ++ .../llama3-70b/r768/result_summary.json | 29 ++ .../llama3-70b/r768/system_desc.json | 61 +++ .../results/sys_8gpu_sglang/system_power.json | 44 ++ .../submissions/sub_h/src/sglang/README.md | 3 + .../submissions/sub_i/docs/.gitkeep | 0 .../deepseek-r1/r1/accuracy_results.json | 11 + .../sys_h200_8gpu/deepseek-r1/r1/config.yaml | 1 + .../sys_h200_8gpu/deepseek-r1/r1/point.yaml | 49 ++ .../deepseek-r1/r1/result_summary.json | 29 ++ .../deepseek-r1/r1/system_desc.json | 62 +++ .../deepseek-r1/r128/accuracy_results.json | 11 + .../deepseek-r1/r128/config.yaml | 1 + .../sys_h200_8gpu/deepseek-r1/r128/point.yaml | 49 ++ .../deepseek-r1/r128/result_summary.json | 29 ++ .../deepseek-r1/r128/system_desc.json | 62 +++ .../deepseek-r1/r16/accuracy_results.json | 11 + .../sys_h200_8gpu/deepseek-r1/r16/config.yaml | 1 + .../sys_h200_8gpu/deepseek-r1/r16/point.yaml | 49 ++ .../deepseek-r1/r16/result_summary.json | 29 ++ .../deepseek-r1/r16/system_desc.json | 62 +++ .../deepseek-r1/r2/accuracy_results.json | 11 + .../sys_h200_8gpu/deepseek-r1/r2/config.yaml | 1 + .../sys_h200_8gpu/deepseek-r1/r2/point.yaml | 49 ++ .../deepseek-r1/r2/result_summary.json | 29 ++ .../deepseek-r1/r2/system_desc.json | 62 +++ .../deepseek-r1/r256/accuracy_results.json | 11 + .../deepseek-r1/r256/config.yaml | 1 + .../sys_h200_8gpu/deepseek-r1/r256/point.yaml | 49 ++ .../deepseek-r1/r256/result_summary.json | 29 ++ .../deepseek-r1/r256/system_desc.json | 62 +++ .../deepseek-r1/r32/accuracy_results.json | 11 + .../sys_h200_8gpu/deepseek-r1/r32/config.yaml | 1 + .../sys_h200_8gpu/deepseek-r1/r32/point.yaml | 49 ++ .../deepseek-r1/r32/result_summary.json | 29 ++ .../deepseek-r1/r32/system_desc.json | 62 +++ .../deepseek-r1/r4/accuracy_results.json | 11 + .../sys_h200_8gpu/deepseek-r1/r4/config.yaml | 1 + .../sys_h200_8gpu/deepseek-r1/r4/point.yaml | 49 ++ .../deepseek-r1/r4/result_summary.json | 29 ++ .../deepseek-r1/r4/system_desc.json | 62 +++ .../deepseek-r1/r512/accuracy_results.json | 11 + .../deepseek-r1/r512/config.yaml | 1 + .../sys_h200_8gpu/deepseek-r1/r512/point.yaml | 50 ++ .../deepseek-r1/r512/result_summary.json | 29 ++ .../deepseek-r1/r512/system_desc.json | 62 +++ .../deepseek-r1/r64/accuracy_results.json | 11 + .../sys_h200_8gpu/deepseek-r1/r64/config.yaml | 1 + .../sys_h200_8gpu/deepseek-r1/r64/point.yaml | 49 ++ .../deepseek-r1/r64/result_summary.json | 29 ++ .../deepseek-r1/r64/system_desc.json | 62 +++ .../deepseek-r1/r8/accuracy_results.json | 11 + .../sys_h200_8gpu/deepseek-r1/r8/config.yaml | 1 + .../sys_h200_8gpu/deepseek-r1/r8/point.yaml | 49 ++ .../deepseek-r1/r8/result_summary.json | 29 ++ .../deepseek-r1/r8/system_desc.json | 62 +++ .../results/sys_h200_8gpu/system_power.json | 44 ++ .../sub_i/src/tensorrt-llm/README.md | 3 + .../submissions/sub_j/docs/.gitkeep | 0 .../deepseek-r1/r1024/accuracy_results.json | 11 + .../deepseek-r1/r1024/config.yaml | 1 + .../deepseek-r1/r1024/point.yaml | 49 ++ .../deepseek-r1/r1024/result_summary.json | 29 ++ .../deepseek-r1/r1024/system_desc.json | 61 +++ .../deepseek-r1/r128/accuracy_results.json | 11 + .../deepseek-r1/r128/config.yaml | 1 + .../deepseek-r1/r128/point.yaml | 49 ++ .../deepseek-r1/r128/result_summary.json | 29 ++ .../deepseek-r1/r128/system_desc.json | 61 +++ .../deepseek-r1/r16384/accuracy_results.json | 11 + .../deepseek-r1/r16384/config.yaml | 1 + .../deepseek-r1/r16384/point.yaml | 50 ++ .../deepseek-r1/r16384/result_summary.json | 29 ++ .../deepseek-r1/r16384/system_desc.json | 61 +++ .../deepseek-r1/r2048/accuracy_results.json | 11 + .../deepseek-r1/r2048/config.yaml | 1 + .../deepseek-r1/r2048/point.yaml | 49 ++ .../deepseek-r1/r2048/result_summary.json | 29 ++ .../deepseek-r1/r2048/system_desc.json | 61 +++ .../deepseek-r1/r256/accuracy_results.json | 11 + .../deepseek-r1/r256/config.yaml | 1 + .../deepseek-r1/r256/point.yaml | 49 ++ .../deepseek-r1/r256/result_summary.json | 29 ++ .../deepseek-r1/r256/system_desc.json | 61 +++ .../deepseek-r1/r32/accuracy_results.json | 11 + .../deepseek-r1/r32/config.yaml | 1 + .../deepseek-r1/r32/point.yaml | 49 ++ .../deepseek-r1/r32/result_summary.json | 29 ++ .../deepseek-r1/r32/system_desc.json | 61 +++ .../deepseek-r1/r4096/accuracy_results.json | 11 + .../deepseek-r1/r4096/config.yaml | 1 + .../deepseek-r1/r4096/point.yaml | 49 ++ .../deepseek-r1/r4096/result_summary.json | 29 ++ .../deepseek-r1/r4096/system_desc.json | 61 +++ .../deepseek-r1/r512/accuracy_results.json | 11 + .../deepseek-r1/r512/config.yaml | 1 + .../deepseek-r1/r512/point.yaml | 49 ++ .../deepseek-r1/r512/result_summary.json | 29 ++ .../deepseek-r1/r512/system_desc.json | 61 +++ .../deepseek-r1/r64/accuracy_results.json | 11 + .../deepseek-r1/r64/config.yaml | 1 + .../deepseek-r1/r64/point.yaml | 49 ++ .../deepseek-r1/r64/result_summary.json | 29 ++ .../deepseek-r1/r64/system_desc.json | 61 +++ .../deepseek-r1/r8192/accuracy_results.json | 11 + .../deepseek-r1/r8192/config.yaml | 1 + .../deepseek-r1/r8192/point.yaml | 49 ++ .../deepseek-r1/r8192/result_summary.json | 29 ++ .../deepseek-r1/r8192/system_desc.json | 61 +++ .../results/sys_gb300_72gpu/system_power.json | 44 ++ .../sub_j/src/tensorrt-llm/README.md | 3 + .../valid_standardized/docs/.gitkeep | 0 .../valid_standardized/docs/calibration.adoc | 5 + .../llama3_1-8b/r1000/accuracy_results.json | 13 + .../llama3_1-8b/r1000/config.yaml | 1 + .../llama3_1-8b/r1000/point.yaml | 50 ++ .../llama3_1-8b/r1000/result_summary.json | 29 ++ .../llama3_1-8b/r1000/system_desc.json | 63 +++ .../llama3_1-8b/r16/accuracy_results.json | 13 + .../llama3_1-8b/r16/config.yaml | 1 + .../llama3_1-8b/r16/point.yaml | 49 ++ .../llama3_1-8b/r16/result_summary.json | 29 ++ .../llama3_1-8b/r16/system_desc.json | 63 +++ .../llama3_1-8b/r20/accuracy_results.json | 13 + .../llama3_1-8b/r20/config.yaml | 1 + .../llama3_1-8b/r20/point.yaml | 49 ++ .../llama3_1-8b/r20/result_summary.json | 29 ++ .../llama3_1-8b/r20/system_desc.json | 63 +++ .../llama3_1-8b/r256/accuracy_results.json | 13 + .../llama3_1-8b/r256/config.yaml | 1 + .../llama3_1-8b/r256/point.yaml | 49 ++ .../llama3_1-8b/r256/result_summary.json | 29 ++ .../llama3_1-8b/r256/system_desc.json | 63 +++ .../llama3_1-8b/r512/accuracy_results.json | 13 + .../llama3_1-8b/r512/config.yaml | 1 + .../llama3_1-8b/r512/point.yaml | 49 ++ .../llama3_1-8b/r512/result_summary.json | 29 ++ .../llama3_1-8b/r512/system_desc.json | 63 +++ .../llama3_1-8b/r768/accuracy_results.json | 13 + .../llama3_1-8b/r768/config.yaml | 1 + .../llama3_1-8b/r768/point.yaml | 49 ++ .../llama3_1-8b/r768/result_summary.json | 29 ++ .../llama3_1-8b/r768/system_desc.json | 63 +++ .../llama3_1-8b/r88/accuracy_results.json | 13 + .../llama3_1-8b/r88/config.yaml | 1 + .../llama3_1-8b/r88/point.yaml | 49 ++ .../llama3_1-8b/r88/result_summary.json | 29 ++ .../llama3_1-8b/r88/system_desc.json | 63 +++ .../results/acme_h100x8_001/system_power.json | 44 ++ .../src/llama3_1-8b/.gitkeep | 0 .../src/llama3_1-8b/README.md | 3 + .../valid_standardized/src/vllm/README.md | 3 + tests/unit/validation/test_artifacts.py | 25 + tests/unit/validation/test_bindings.py | 91 ++++ tests/unit/validation/test_engine.py | 323 ++++++++++++ tests/unit/validation/test_evidence.py | 18 + .../unit/validation/test_policy_contracts.py | 143 +++++- .../validation/test_review_regressions.py | 193 +++++++ .../validation/test_special_evaluators.py | 477 ++++++++++++++++++ 570 files changed, 19823 insertions(+), 2 deletions(-) create mode 100644 src/inference_endpoint/validation/api.py create mode 100644 src/inference_endpoint/validation/bindings.py create mode 100644 src/inference_endpoint/validation/checks/__init__.py create mode 100644 src/inference_endpoint/validation/checks/accuracy.py create mode 100644 src/inference_endpoint/validation/checks/collections.py create mode 100644 src/inference_endpoint/validation/checks/datasets.py create mode 100644 src/inference_endpoint/validation/checks/drafters.py create mode 100644 src/inference_endpoint/validation/checks/helpers.py create mode 100644 src/inference_endpoint/validation/checks/metrics.py create mode 100644 src/inference_endpoint/validation/checks/power.py create mode 100644 src/inference_endpoint/validation/checks/seeds.py create mode 100644 src/inference_endpoint/validation/checks/steady_state.py create mode 100644 src/inference_endpoint/validation/checks/structure.py create mode 100644 src/inference_endpoint/validation/checks/warmup.py create mode 100644 src/inference_endpoint/validation/engine.py create mode 100644 src/inference_endpoint/validation/evaluator_base.py create mode 100644 src/inference_endpoint/validation/outcomes.py create mode 100644 tests/fixtures/validation/submissions/invalid_submission/docs/.gitkeep create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/config.yaml create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/point.yaml create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/result_summary.json create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/system_desc.json create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/config.yaml create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/point.yaml create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/result_summary.json create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/system_desc.json create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/config.yaml create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/point.yaml create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/result_summary.json create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/system_desc.json create mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/bad_system/system_power.json create mode 100644 tests/fixtures/validation/submissions/invalid_submission/src/custom/README.md create mode 100644 tests/fixtures/validation/submissions/sub_a/docs/.gitkeep create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/system_power.json create mode 100644 tests/fixtures/validation/submissions/sub_a/src/vllm/README.md create mode 100644 tests/fixtures/validation/submissions/sub_b/docs/.gitkeep create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/system_power.json create mode 100644 tests/fixtures/validation/submissions/sub_b/src/vllm/README.md create mode 100644 tests/fixtures/validation/submissions/sub_c/docs/.gitkeep create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/system_power.json create mode 100644 tests/fixtures/validation/submissions/sub_c/src/vllm/README.md create mode 100644 tests/fixtures/validation/submissions/sub_d/docs/.gitkeep create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/system_power.json create mode 100644 tests/fixtures/validation/submissions/sub_d/src/vllm/README.md create mode 100644 tests/fixtures/validation/submissions/sub_e/docs/.gitkeep create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/system_power.json create mode 100644 tests/fixtures/validation/submissions/sub_e/src/vllm/README.md create mode 100644 tests/fixtures/validation/submissions/sub_f/docs/.gitkeep create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/system_power.json create mode 100644 tests/fixtures/validation/submissions/sub_f/src/vllm/README.md create mode 100644 tests/fixtures/validation/submissions/sub_g/docs/.gitkeep create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/system_power.json create mode 100644 tests/fixtures/validation/submissions/sub_g/src/vllm/README.md create mode 100644 tests/fixtures/validation/submissions/sub_h/docs/.gitkeep create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/system_power.json create mode 100644 tests/fixtures/validation/submissions/sub_h/src/sglang/README.md create mode 100644 tests/fixtures/validation/submissions/sub_i/docs/.gitkeep create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/system_power.json create mode 100644 tests/fixtures/validation/submissions/sub_i/src/tensorrt-llm/README.md create mode 100644 tests/fixtures/validation/submissions/sub_j/docs/.gitkeep create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/config.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/point.yaml create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/result_summary.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/system_desc.json create mode 100644 tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/system_power.json create mode 100644 tests/fixtures/validation/submissions/sub_j/src/tensorrt-llm/README.md create mode 100644 tests/fixtures/validation/submissions/valid_standardized/docs/.gitkeep create mode 100644 tests/fixtures/validation/submissions/valid_standardized/docs/calibration.adoc create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/config.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/point.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/result_summary.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/system_desc.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/config.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/point.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/result_summary.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/system_desc.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/config.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/point.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/result_summary.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/system_desc.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/config.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/point.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/result_summary.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/system_desc.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/config.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/point.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/result_summary.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/system_desc.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/config.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/point.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/result_summary.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/system_desc.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/accuracy_results.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/config.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/point.yaml create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/result_summary.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/system_desc.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/system_power.json create mode 100644 tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/.gitkeep create mode 100644 tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/README.md create mode 100644 tests/fixtures/validation/submissions/valid_standardized/src/vllm/README.md create mode 100644 tests/unit/validation/test_bindings.py create mode 100644 tests/unit/validation/test_engine.py create mode 100644 tests/unit/validation/test_review_regressions.py create mode 100644 tests/unit/validation/test_special_evaluators.py diff --git a/.gitignore b/.gitignore index 1c4ab65a5..562a63e7c 100644 --- a/.gitignore +++ b/.gitignore @@ -184,6 +184,8 @@ temp/ tmp/ data/ results/ +!tests/fixtures/validation/submissions/**/results/ +!tests/fixtures/validation/submissions/**/results/** outputs/ # Bundled example datasets are intentionally committed (git-LFS); the broad diff --git a/src/inference_endpoint/validation/__init__.py b/src/inference_endpoint/validation/__init__.py index 922321be9..adb63ec4a 100644 --- a/src/inference_endpoint/validation/__init__.py +++ b/src/inference_endpoint/validation/__init__.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Versioned submission-validation policies and typed check planning.""" +from .api import CheckResult, Report, SubmissionChecker, validate_submission from .conditions import Conditions, Context from .loader import bundled_policy_path, load_policy from .models import Policy, Rule @@ -10,6 +11,7 @@ __all__ = [ "CheckPlan", + "CheckResult", "Conditions", "Context", "Decision", @@ -17,8 +19,11 @@ "PlannedCheck", "Policy", "Rule", + "Report", "Scope", + "SubmissionChecker", "bundled_policy_path", "load_policy", "plan_checks", + "validate_submission", ] diff --git a/src/inference_endpoint/validation/api.py b/src/inference_endpoint/validation/api.py new file mode 100644 index 000000000..5db06bbd1 --- /dev/null +++ b/src/inference_endpoint/validation/api.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Validate classified submission artifacts against a selected cohort policy.""" + +from dataclasses import dataclass +from os import PathLike + +from .engine import execute_submission +from .loader import bundled_policy_path, load_policy +from .models import Policy +from .results import CheckResult, Report + +__all__ = ["CheckResult", "Report", "SubmissionChecker", "validate_submission"] + + +@dataclass(frozen=True) +class SubmissionChecker: + """Reusable checker configuration; each run parses and evaluates fresh evidence.""" + + submission_path: PathLike[str] + policy: Policy | PathLike[str] | None = None + + def run(self) -> Report: + policy = ( + self.policy + if isinstance(self.policy, Policy) + else load_policy(self.policy or bundled_policy_path()) + ) + return execute_submission( + self.submission_path, + policy, + ) + + +def validate_submission( + path: PathLike[str], + *, + policy: Policy | PathLike[str] | None = None, +) -> Report: + """Load a cohort policy, classify artifacts, and evaluate its applicable checks.""" + return SubmissionChecker( + submission_path=path, + policy=policy, + ).run() diff --git a/src/inference_endpoint/validation/bindings.py b/src/inference_endpoint/validation/bindings.py new file mode 100644 index 000000000..acbd846ad --- /dev/null +++ b/src/inference_endpoint/validation/bindings.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Bind model checkpoints and client revisions to published policy approvals.""" + +import re + +from .artifacts import Artifacts +from .evaluator_base import Evaluator +from .operations import BindingMatch +from .outcomes import result +from .planner import PlannedCheck +from .results import CheckResult +from .vocabulary import CheckKind + + +class ArtifactBinding(Evaluator, kind=CheckKind.ARTIFACT_BINDING): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + matching = BindingMatch(check.rule.requirements["matching"]) + findings = [] + for point in artifacts.members(check): + if matching is BindingMatch.CHECKPOINT: + addresses = check.rule.requirements["sources"] + repository, revision = ( + artifacts.resolve(address, check, point) for address in addresses + ) + model = point.context.model_id + catalog = artifacts.resolve(check.rule.requirements["catalog"], check) + approved = catalog.get(model, ()) if catalog is not None else () + if not approved: + findings.append( + result( + check, + False, + f"No approved checkpoint catalog for {model}", + point=point, + blocked=True, + ) + ) + elif not repository or not revision: + findings.append( + result( + check, + False, + "Checkpoint repository and revision are required", + point=point, + ) + ) + else: + matches = any( + entry["repository"] == repository + and entry["revision"] == revision + for entry in approved + ) + findings.append( + result( + check, + matches, + "Checkpoint matches model approval" + if matches + else "Checkpoint is not approved for this model", + point=point, + ) + ) + continue + revision = artifacts.resolve( + check.rule.requirements["source"], check, point + ) + if not isinstance(revision, str) or not re.fullmatch( + r"[0-9a-f]{40}", revision + ): + findings.append( + result( + check, + False, + "Client revision must be a full lowercase Git SHA-1", + point=point, + ) + ) + continue + approved = artifacts.resolve(check.rule.requirements["catalog"], check) + if not approved: + findings.append( + result( + check, + False, + "Client revision approval list is empty", + point=point, + blocked=True, + ) + ) + continue + accepted = revision in approved + findings.append( + result( + check, + accepted, + "Client revision is approved" + if accepted + else "Client revision is not approved", + point=point, + ) + ) + return findings + + +class CatalogIntegrity(Evaluator, kind=CheckKind.CATALOG_INTEGRITY): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + address = check.rule.requirements["catalog"] + catalog_name = ( + "seed_sets" + if address == "context.cohorts.seed_sets" + else "approved_sped_decode_heads" + ) + problem = artifacts.catalogs.errors.get(catalog_name) + catalog = artifacts.resolve(address, check) + if problem is not None or catalog is None: + return [ + result( + check, + False, + f"Required catalog is unavailable: {problem or address}", + blocked=True, + ) + ] + return [result(check, True, f"Required catalog is available: {address}")] diff --git a/src/inference_endpoint/validation/checks/__init__.py b/src/inference_endpoint/validation/checks/__init__.py new file mode 100644 index 000000000..4b660d653 --- /dev/null +++ b/src/inference_endpoint/validation/checks/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Register built-in callable evaluators by their policy kind.""" + +from . import accuracy, drafters, power, seeds, steady_state, warmup + +__all__ = ["accuracy", "drafters", "power", "seeds", "steady_state", "warmup"] diff --git a/src/inference_endpoint/validation/checks/accuracy.py b/src/inference_endpoint/validation/checks/accuracy.py new file mode 100644 index 000000000..23b4698aa --- /dev/null +++ b/src/inference_endpoint/validation/checks/accuracy.py @@ -0,0 +1,470 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy-driven accuracy checks.""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Mapping +from typing import Any, cast + +from pydantic import JsonValue + +from inference_endpoint.config.schema import LoadPatternType + +from ..artifacts import Artifacts, PointArtifacts +from ..evaluator_base import Evaluator +from ..operations import AccuracyOperation +from ..planner import PlannedCheck +from ..results import CheckResult, Severity +from ..types import AccuracyKind, OfflineMode +from ..vocabulary import CheckKind +from .helpers import finding, number, unsupported, validate_modes, warning + + +def accuracy_entries( + point: PointArtifacts, +) -> tuple[dict[str, dict[str, JsonValue]], dict[str, dict[str, float]]]: + """Read the parser's normalized dataset mapping and numeric metric scores.""" + if point.accuracy is None: + return {}, {} + return point.accuracy.root, point.accuracy.metric_scores() + + +def dataset_matches(name: str, required: str) -> bool: + return ( + name == required + or name.split("::", 1)[0] == required + or name.startswith(required + "_") + ) + + +def weighted_mean(pairs: list[tuple[float, object]]) -> float: + numeric_pairs = [ + (value, weight) for value, weight in pairs if number(weight) and weight > 0 + ] + if len(numeric_pairs) == len(pairs): + return sum(value * weight for value, weight in numeric_pairs) / sum( + weight for _, weight in numeric_pairs + ) + return sum(value for value, _ in pairs) / len(pairs) + + +def accuracy_band(artifacts: Artifacts, point: PointArtifacts) -> str | None: + if ( + point.concurrency is not None + and 1 + <= point.concurrency + <= artifacts.policy.catalogs["regions"]["ultra_low_maximum"] + ): + return "ultra_low_concurrency" + return artifacts.region(point) + + +class AccuracyPresenceEvaluator(Evaluator, kind=CheckKind.ACCURACY_PRESENCE): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + accepted = check.rule.requirements["accept"] + curves = set() + for point in artifacts.members(check): + root, _ = accuracy_entries(point) + standalone = (point.path / "accuracy_results.json").exists() + if root and ( + (standalone and "standalone_accuracy_results" in accepted) + or (not standalone and "embedded_nonempty_accuracy_scores" in accepted) + ): + curves.add(point.curve) + minimum = check.rule.requirements["minimum_models"] + return [ + finding( + check, + len(curves) >= minimum, + f"Accuracy results present for {len(curves)} models; requires {minimum}", + ) + ] + + +class AccuracyCoverageEvaluator(Evaluator, kind=CheckKind.ACCURACY_COVERAGE): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + params = check.rule.requirements + invalid = validate_modes(check) + if invalid is not None: + return invalid + bands = artifacts.resolve(params["mandatory_bands"], check) + counts: defaultdict[str | None, int] = defaultdict(int) + offline = False + for point in artifacts.members(check): + root, _ = accuracy_entries(point) + if not root: + continue + band = artifacts.region(point) + if band != "margin" or params.get("margin_counts"): + counts[band] += 1 + if ( + point.concurrency is not None + and 1 + <= point.concurrency + <= artifacts.policy.catalogs["regions"]["ultra_low_maximum"] + ): + counts["ultra_low_concurrency"] += 1 + offline |= point.get_config("offline") in ( + OfflineMode.DEDICATED, + OfflineMode.ELECTED, + ) + output = [ + finding( + check, + counts[band] >= params["minimum_results_per_band"], + f"Accuracy band {band}: {counts[band]} results; requires {params['minimum_results_per_band']}", + ) + for band in bands + ] + patterns = { + p.get_config("runtime_settings.load_pattern") + for p in artifacts.members(check) + if p.get_config("offline") != OfflineMode.DEDICATED + } + if ( + params.get("single_turn_requires_offline_accuracy") + and LoadPatternType.AGENTIC_INFERENCE not in patterns + ): + output.append( + finding( + check, + offline, + "Dedicated offline accuracy result present" + if offline + else "Dedicated offline accuracy result missing", + ) + ) + return output + + +class AccuracyGateEvaluator(Evaluator, kind=CheckKind.ACCURACY_GATE): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + params = check.rule.requirements + invalid = validate_modes(check) + if invalid is not None: + return invalid + try: + operation = AccuracyOperation(cast(str, params.get("operation"))) + except ValueError: + return unsupported(check) + model_catalog = artifacts.resolve(params["models"], check) + model = ( + model_catalog.get(check.subject.model_id, {}) + if isinstance(model_catalog, Mapping) + else {} + ) + profile = model.get("accuracy") + if not isinstance(profile, Mapping): + return [ + warning( + check, f"No published accuracy profile for {check.subject.model_id}" + ) + ] + points = artifacts.members(check) + if operation is AccuracyOperation.AGENTIC_PROFILE_AVAILABLE: + return [ + finding( + check, + profile.get("kind") == AccuracyKind.AGENTIC, + f"Accuracy profile kind: {profile.get('kind')}", + severity=Severity.WARNING, + ) + ] + if operation is AccuracyOperation.PER_POINT_RANGE: + bounds = artifacts.resolve(params["bounds"], check) + if not isinstance(bounds, (tuple, list)) or len(bounds) != 2: + return unsupported(check, "Accuracy range catalog unavailable") + output = [] + for point in points: + value = artifacts.resolve(params["source"], check, point) + if not number(value): + output.append( + warning( + check, + "Full-run output sequence length is unavailable", + point, + ) + ) + continue + low, high = bounds + passed = ( + low <= value <= high + if params.get("inclusive") + else low < value < high + ) + output.append( + finding( + check, + passed, + f"Full-run output sequence length {value}; required range [{low}, {high}]", + point=point, + ) + ) + return output + if operation in ( + AccuracyOperation.PER_POINT_FRACTION, + AccuracyOperation.MEAN_OF_BAND_MEANS, + ): + return self._agentic(check, artifacts, points, params, operation) + return self._single_turn(check, artifacts, points, params, profile, operation) + + def _agentic( + self, + check: PlannedCheck, + artifacts: Artifacts, + points: list[PointArtifacts], + params: Mapping[str, Any], + operation: AccuracyOperation, + ) -> list[CheckResult]: + minimum = artifacts.resolve(params["minimum"], check) + if not number(minimum): + return unsupported(check, "Accuracy minimum unavailable") + output = [] + by_band: defaultdict[str | None, list[float]] = defaultdict(list) + for point in points: + root, scores = accuracy_entries(point) + selected = [ + (name, metrics) + for name, metrics in scores.items() + if dataset_matches(name, params["dataset"]) + ] + if not selected: + if operation is AccuracyOperation.PER_POINT_FRACTION: + output.append( + warning( + check, f"Missing {params['dataset']} accuracy result", point + ) + ) + continue + values = [] + for name, metrics in selected: + value = metrics.get("score", next(iter(metrics.values()), None)) + lo, hi = params["input_range"] + if not number(value) or not lo <= value <= hi: + output.append( + finding( + check, + False, + f"Accuracy score {value!r} for {name} is outside [{lo}, {hi}]", + point=point, + ) + ) + continue + values.append(value * params["output_scale"]) + if not values: + continue + average = sum(values) / len(values) + if operation is AccuracyOperation.PER_POINT_FRACTION: + output.append( + finding( + check, + average >= minimum, + f"Accuracy {average:g}%; minimum {minimum:g}%", + point=point, + ) + ) + else: + by_band[accuracy_band(artifacts, point)].append(average) + if operation is AccuracyOperation.MEAN_OF_BAND_MEANS: + bands = artifacts.resolve(params["bands"], check) + available = [band for band in bands if by_band[band]] + if len(available) < params["required_band_count"]: + output.append( + warning( + check, + f"SWE-bench accuracy has {len(available)} populated bands; requires {params['required_band_count']}", + ) + ) + for band in available: + if len(by_band[band]) > 1: + output.append( + warning( + check, + f"Averaging {len(by_band[band])} SWE-bench results in {band}", + ) + ) + if available: + means = [sum(by_band[band]) / len(by_band[band]) for band in available] + mean = sum(means) / len(means) + output.append( + finding( + check, + mean >= minimum, + f"SWE-bench mean of band means {mean:g}%; minimum {minimum:g}%", + ) + ) + else: + output.append(warning(check, "No SWE-bench accuracy results available")) + return output + + def _single_turn( + self, + check: PlannedCheck, + artifacts: Artifacts, + points: list[PointArtifacts], + params: Mapping[str, Any], + profile: Mapping[str, Any], + operation: AccuracyOperation, + ) -> list[CheckResult]: + required = artifacts.resolve(params["required_datasets"], check) + datasets = artifacts.resolve(params["dataset_catalog"], check) + metrics = profile.get("metrics", {}) + if not required or not isinstance(datasets, Mapping): + return unsupported(check, "Required accuracy datasets unavailable") + if ( + operation is AccuracyOperation.SINGLE_TURN_METRICS + and params.get("aggregation") != "sample_weighted_mean" + ): + return unsupported(check, "Unsupported accuracy aggregation") + output = [] + for point in points: + root, scores = accuracy_entries(point) + if not root: + continue + selected = { + name: entry + for name, entry in root.items() + if any(dataset_matches(name, ds) for ds in required) + } + missing = [ + ds + for ds in required + if not any(dataset_matches(name, ds) for name in selected) + ] + if missing: + output.append( + finding( + check, + False, + f"Required accuracy datasets missing: {', '.join(missing)}", + point=point, + ) + ) + continue + if operation is AccuracyOperation.ISSUED_COUNT: + for ds in required: + threshold = datasets.get(ds, {}).get(params["threshold"]) + if not number(threshold): + output.append( + finding( + check, + False, + f"Accuracy sample threshold unavailable for {ds}", + point=point, + blocked=True, + ) + ) + continue + counts = [] + for name, entry in selected.items(): + if not dataset_matches(name, ds): + continue + count = entry.get("num_samples") + repeats = entry.get("n_repeats", params["missing_repeats"]) + if number(count) and number(repeats): + counts.append( + count * max(repeats, params["minimum_repeats"]) + ) + if counts: + total = sum(counts) + output.append( + finding( + check, + total >= threshold, + f"Dataset {ds}: {total:g} issued accuracy samples; requires {threshold:g}", + point=point, + ) + ) + continue + pairs = defaultdict(list) + for name, entry in selected.items(): + weight = entry.get("num_samples") + for metric, value in scores.get(name, {}).items(): + pairs[ + metric.lower() + if params.get("metric_matching") == "case_insensitive" + else metric + ].append((value, weight)) + aggregate = { + metric: weighted_mean(values) for metric, values in pairs.items() + } + if ( + set(aggregate) == {"score"} + and len(metrics) == 1 + and params.get("unnamed_scalar") == "sole_metric_only_with_warning" + ): + name = next(iter(metrics)) + aggregate[name] = aggregate.pop("score") + output.append( + warning( + check, + f"Unnamed scalar accuracy is interpreted as sole metric {name}", + point, + ) + ) + for name, bounds in metrics.items(): + score = aggregate.get( + name.lower() + if params.get("metric_matching") == "case_insensitive" + else name + ) + if score is None: + if params.get("on_missing_metric") != "skip": + output.append( + finding( + check, + False, + f"Accuracy metric {name} missing", + point=point, + ) + ) + continue + reference = bounds.get("reference", bounds.get("minimum_reference")) + upper_reference = bounds.get( + "reference", bounds.get("maximum_reference") + ) + lower = ( + reference * bounds["minimum_multiplier"] + if reference is not None and "minimum_multiplier" in bounds + else None + ) + upper = ( + upper_reference * bounds["maximum_multiplier"] + if upper_reference is not None and "maximum_multiplier" in bounds + else None + ) + if lower is None and upper is None: + output.append( + finding( + check, + False, + f"Accuracy bounds unavailable for {name}", + point=point, + blocked=True, + ) + ) + continue + if ( + params.get("fraction_conversion") + == "legacy_score_and_threshold_heuristic" + and 0 <= score <= 1 + and lower is not None + and lower > 1 + ): + score *= 100 + passed = ( + number(score) + and (lower is None or score >= lower) + and (upper is None or score <= upper) + ) + output.append( + finding( + check, + passed, + f"Accuracy metric {name}={score:g}; accepted bounds [{lower}, {upper}]", + point=point, + ) + ) + return output diff --git a/src/inference_endpoint/validation/checks/collections.py b/src/inference_endpoint/validation/checks/collections.py new file mode 100644 index 000000000..851e146bf --- /dev/null +++ b/src/inference_endpoint/validation/checks/collections.py @@ -0,0 +1,265 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy evaluators for collections.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from pathlib import Path + +from inference_endpoint.config.schema import LoadPatternType + +from ..artifacts import Artifacts, PointArtifacts +from ..evaluator_base import Evaluator +from ..operations import OfflineOperation, RegionPlacementOperation +from ..outcomes import result +from ..planner import PlannedCheck +from ..results import CheckResult, Severity +from ..types import AccuracyKind, OfflineMode +from ..vocabulary import CheckKind + + +class CollectionSize(Evaluator, kind=CheckKind.COLLECTION_SIZE): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + count = len(artifacts.resolve(req["source"], check) or []) + minimum = req.get("minimum", 0) + maximum = req.get("maximum", math.inf) + members = artifacts.members(check) + for override in req.get("overrides", []): + when = override["when"] + agentic = any( + p.get_config("runtime_settings.load_pattern") + == LoadPatternType.AGENTIC_INFERENCE + for p in members + ) + if ( + when.get("curve_type") == AccuracyKind.SINGLE_TURN + and not agentic + and any( + p.get_config("offline") == when.get("has_offline") for p in members + ) + ): + minimum = override.get("minimum", minimum) + return [ + result( + check, + minimum <= count <= maximum, + f"Collection has {count} members; allowed [{minimum}, {maximum}]", + ) + ] + + +class RegionBasis(Evaluator, kind=CheckKind.REGION_BASIS): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + if check.subject.id in artifacts.derived.region_errors: + return [ + result( + check, + False, + artifacts.derived.region_errors[check.subject.id], + blocked=True, + ) + ] + points = artifacts.members(check) + concurrencies = [p.concurrency for p in points if p.concurrency is not None] + req = check.rule.requirements + if not concurrencies: + return [result(check, False, "No readable concurrency values")] + finding = result( + check, + True, + f"C_min={min(min(concurrencies), req['upper_clamp'])} from {len(concurrencies)} points", + ) + if ( + len(concurrencies) < len(points) + and req.get("on_partial_parse") == "warning" + ): + finding = finding.model_copy(update={"severity": Severity.WARNING}) + return [finding] + + +class RegionBoundaries(Evaluator, kind=CheckKind.REGION_BOUNDARIES): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + cmax = artifacts.resolve("curve.max_supported_concurrency", check) + points = artifacts.members(check) + cs = [p.concurrency for p in points if p.concurrency is not None] + cmin = min(min(cs), req["require_c_min"][1]) if cs else None + valid = ( + cmin is not None + and req["require_c_min"][0] <= cmin <= req["require_c_min"][1] + and isinstance(cmax, (int, float)) + and not isinstance(cmax, bool) + and cmax > req["require_c_max_greater_than"] + and check.subject.id in artifacts.derived.regions + ) + return [ + result( + check, + valid, + f"Region basis C_min={cmin}, C_max={cmax}; boundaries {artifacts.derived.regions.get(check.subject.id)}", + ) + ] + + +class Coverage(Evaluator, kind=CheckKind.COVERAGE): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + band = req["band"] + members = artifacts.members(check) + + def in_band(point: PointArtifacts) -> bool: + if point.get_config("offline") == OfflineMode.DEDICATED: + return False + if isinstance(band, Mapping): + return ( + point.concurrency is not None + and band["minimum"] <= point.concurrency <= band["maximum"] + ) + actual = artifacts.region(point) + return ( + actual == band + or band == "high_concurrency" + and actual == "margin" + and req.get("margin_counts", False) + ) + + count = sum(in_band(p) for p in members) + return [ + result( + check, + count >= req["minimum_count"], + f"{band}: {count} points; requires {req['minimum_count']}", + ) + ] + + +class RegionPlacement(Evaluator, kind=CheckKind.REGION_PLACEMENT): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + point = artifacts.index.points[check.subject.id] + actual = artifacts.region(point) + try: + operation = RegionPlacementOperation(req["operation"]) + except ValueError: + return [ + result( + check, + False, + f"Unsupported region-placement operation: {req['operation']}", + blocked=True, + ) + ] + if operation is RegionPlacementOperation.IN_RANGE: + return [ + result( + check, + actual is not None + and (req.get("include_margin") or actual != "margin"), + f"Concurrency {point.concurrency} classified as {actual}", + ) + ] + declared = point.get_config("region") + if ( + declared in req.get("ignore_declared", []) + or (actual is None or declared is None) + and req.get("on_missing_or_unclassifiable") == "skip" + ): + return [] + return [ + result( + check, declared == actual, f"Declared {declared!r}; computed {actual!r}" + ) + ] + + +class Offline(Evaluator, kind=CheckKind.OFFLINE): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + points = artifacts.members(check) + cmax = artifacts.resolve("curve.max_supported_concurrency", check) + try: + operation = OfflineOperation(req["operation"]) + except ValueError: + return [ + result( + check, + False, + f"Unsupported Offline operation: {req['operation']}", + blocked=True, + ) + ] + if operation is OfflineOperation.DECLARATION_COUNT: + elected_concurrency = artifacts.resolve(req["elected_must_equal"], check) + agentic = any( + p.get_config("runtime_settings.load_pattern") + == LoadPatternType.AGENTIC_INFERENCE + for p in points + ) + offline = [ + p + for p in points + if p.get_config("offline") + in (OfflineMode.ELECTED, OfflineMode.DEDICATED) + ] + expected = req["agentic_count" if agentic else "single_turn_count"] + findings = [ + result( + check, + len(offline) == expected, + f"{len(offline)} Offline points; expected {expected}", + ) + ] + findings.extend( + result( + check, + p.concurrency == elected_concurrency, + f"Elected concurrency {p.concurrency}; expected {elected_concurrency}", + point=p, + ) + for p in offline + if p.get_config("offline") == OfflineMode.ELECTED + ) + return findings + floor = artifacts.resolve(req["concurrency_floor"], check) + reference = next( + ( + p + for p in artifacts.index.points.values() + if p.curve == Path(check.subject.id) + and p.concurrency == cmax + and p.get_config("offline") != OfflineMode.DEDICATED + ), + None, + ) + findings = [] + for point in points: + if point.get_config("offline") != OfflineMode.DEDICATED: + continue + findings.append( + result( + check, + floor is not None + and point.concurrency is not None + and point.concurrency >= floor, + f"Offline concurrency {point.concurrency}; floor {floor}", + point=point, + ) + ) + if ( + reference is not None + and reference.throughput is not None + and point.throughput is not None + ): + minimum = reference.throughput * req["throughput_minimum_multiplier"] + findings.append( + result( + check, + point.throughput >= minimum, + f"Offline throughput {point.throughput}; minimum {minimum}", + point=point, + ) + ) + return findings diff --git a/src/inference_endpoint/validation/checks/datasets.py b/src/inference_endpoint/validation/checks/datasets.py new file mode 100644 index 000000000..fe30102eb --- /dev/null +++ b/src/inference_endpoint/validation/checks/datasets.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy evaluators for datasets.""" + +from __future__ import annotations + +from ..artifacts import Artifacts +from ..evaluator_base import Evaluator +from ..operations import DatasetCountOperator +from ..outcomes import result +from ..planner import PlannedCheck +from ..results import CheckResult +from ..types import SampleUnit +from ..vocabulary import CheckKind + + +class DatasetMinimum(Evaluator, kind=CheckKind.DATASET_MINIMUM): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + point = artifacts.index.points[check.subject.id] + dataset = point.get_config("dataset") + catalog = artifacts.resolve(req["catalog"], check) or {} + entry = catalog.get(dataset, {}) + if entry.get("sample_unit", SampleUnit.SAMPLE) not in req.get( + "supported_sample_units", [SampleUnit.SAMPLE] + ): + if req.get("on_unsupported_sample_unit") == "skip": + return [] + return [result(check, False, "Unsupported sample unit", blocked=True)] + minimum = entry.get(req["threshold"]) + if minimum is None: + if req.get("on_unknown_or_null_threshold") == "skip": + return [] + return [result(check, False, "Dataset threshold unavailable", blocked=True)] + actual = artifacts.resolve(req["source"], check) + return [ + result( + check, + isinstance(actual, (int, float)) + and not isinstance(actual, bool) + and actual >= minimum, + f"Completed {actual!r}; {dataset} requires {minimum}", + ) + ] + + +class DatasetCount(Evaluator, kind=CheckKind.DATASET_COUNT): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + entry = (artifacts.resolve(req["catalog"], check) or {}).get(req["dataset"], {}) + expected = entry.get("sample_count") + sources = req.get("sources") or [req["source"]] + findings = [] + if expected is None or expected <= 0: + return [result(check, False, "Dataset count unavailable", blocked=True)] + for source in sources: + actual = artifacts.resolve(source, check) + valid = ( + isinstance(actual, int) + and not isinstance(actual, bool) + and ( + actual == expected + if req["operator"] == DatasetCountOperator.EQUAL + else actual > 0 and actual % expected == 0 + ) + ) + findings.append( + result( + check, + valid, + f"{source}: {actual!r}; expected {req['operator']} {expected}", + ) + ) + return findings + + +class FieldConstraints(Evaluator, kind=CheckKind.FIELD_CONSTRAINTS): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + point = artifacts.index.points[check.subject.id] + if "constraints_by_model" in req: + constraint = ( + artifacts.resolve(req["constraints_by_model"], check) or {} + ).get(check.subject.model_id) + if constraint is None: + return [ + result( + check, False, "Model field constraint unavailable", blocked=True + ) + ] + constraints = {req["target"]: constraint} + else: + constraints = artifacts.resolve(req["constraints"], check) + findings = [] + for field, constraint in constraints.items(): + value = ( + artifacts.resolve(field, check) + if field.startswith("accuracy.") + else point.get_config(field) + ) + valid = ( + value is None + if constraint.get("absent") + else value == constraint.get("equals") + and type(value) is type(constraint.get("equals")) + ) + findings.append( + result(check, valid, f"{field}: {value!r}; expected {dict(constraint)}") + ) + return findings diff --git a/src/inference_endpoint/validation/checks/drafters.py b/src/inference_endpoint/validation/checks/drafters.py new file mode 100644 index 000000000..a81f96e75 --- /dev/null +++ b/src/inference_endpoint/validation/checks/drafters.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy-driven drafters checks.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import cast + +from ..artifacts import Artifacts, nested +from ..cohorts import Cohort +from ..evaluator_base import Evaluator +from ..operations import DrafterOperation +from ..planner import PlannedCheck +from ..results import CheckResult +from ..vocabulary import CheckKind +from .helpers import finding, unsupported, validate_modes, warning + + +class DrafterBindingEvaluator(Evaluator, kind=CheckKind.DRAFTER_BINDING): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + params = check.rule.requirements + invalid = validate_modes(check) + if invalid is not None: + return invalid + try: + operation = DrafterOperation(cast(str, params.get("operation"))) + except ValueError: + return unsupported(check) + catalog = artifacts.resolve(params["catalog"], check) + if catalog is None: + return unsupported(check, "Approved drafter catalog unavailable") + output = [] + for point in artifacts.members(check): + declared = point.get_config("speculative_decoding") + if declared is None: + declared = point.get_config("drafter") + if declared is None: + declared = {} + model = point.context.model_id + matched = None + entries = ( + catalog.get(model, ()) if isinstance(catalog, Mapping) else catalog + ) + for entry in entries: + if isinstance(entry, Mapping): + if ( + entry.get("model") == model + and entry.get("repository") == declared.get("repository") + and entry.get("revision") == declared.get("revision") + ): + matched = entry + break + else: + return unsupported( + check, + "Drafter catalog entries must disclose repository and revision for model identity matching", + ) + if operation is DrafterOperation.MEMBERSHIP: + output.append( + finding( + check, + matched is not None, + "Drafter matches approved model, repository and revision" + if matched + else "Drafter does not match an approved identity", + point=point, + ) + ) + continue + if matched is None and params.get("on_unmatched_drafter") == "skip": + continue + approved = Cohort.parse(nested(matched, "approved_cohort") or "") + target = Cohort.parse(point.get_config("target_cohort") or "") + if approved is None or target is None: + output.append( + warning( + check, + "Drafter approval age cannot be verified without valid approval and target cohorts", + point, + ) + ) + continue + earliest = approved + for _ in range(params["minimum_cohorts"]): + earliest = earliest.next() + output.append( + finding( + check, + target >= earliest, + f"Drafter target {target}; earliest permitted cohort {earliest}", + point=point, + ) + ) + return output diff --git a/src/inference_endpoint/validation/checks/helpers.py b/src/inference_endpoint/validation/checks/helpers.py new file mode 100644 index 000000000..ca99d5a2d --- /dev/null +++ b/src/inference_endpoint/validation/checks/helpers.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared findings, numeric validation, and supported policy execution modes.""" + +from __future__ import annotations + +import math +from typing import TypeGuard + +from pydantic import JsonValue + +from ..artifacts import PointArtifacts +from ..outcomes import result +from ..planner import PlannedCheck +from ..results import CheckResult, Severity + + +def values(value: object) -> list[object]: + return value if isinstance(value, list) else [value] + + +def finding( + check: PlannedCheck, + passed: bool, + message: str, + *, + point: PointArtifacts | None = None, + severity: Severity | None = None, + blocked: bool = False, +) -> CheckResult: + value = result(check, passed, message, point=point, blocked=blocked) + return ( + value.model_copy(update={"severity": severity}) + if severity is not None and not passed and not blocked + else value + ) + + +def warning( + check: PlannedCheck, message: str, point: PointArtifacts | None = None +) -> CheckResult: + return finding(check, False, message, point=point, severity=Severity.WARNING) + + +def unsupported( + check: PlannedCheck, message: str = "Unsupported policy operation" +) -> list[CheckResult]: + return [finding(check, False, message, blocked=True)] + + +def number(value: object) -> TypeGuard[int | float]: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + ) + + +def block_value(point: PointArtifacts, name: str) -> dict[str, JsonValue] | None: + value = getattr(point.config, name, None) + return ( + value.model_dump(mode="json", exclude_unset=True) + if value is not None + else point.get_config(name) + ) + + +def validate_modes(check: PlannedCheck) -> list[CheckResult] | None: + """Reject policy modes whose semantics this parser revision cannot execute.""" + allowed = { + "matching": {"model_and_repository_and_revision"}, + "on_unavailable": {"blocked"}, + "on_legacy_registry_without_cohorts": {"blocked"}, + "on_empty_catalog": {"error"}, + "on_missing_repository_or_revision": {"error"}, + "on_unmatched_drafter": {"skip"}, + "on_unparseable_cohort": {"warning"}, + "on_missing_approval_cohort": {"warning"}, + "on_undeclared_nodes": {"fully_engaged"}, + "declared_total_scaling": {"maximum_engaged_fraction"}, + "switch_scaling": {"engaged_node_fraction"}, + "on_spare_nodes": {"warning"}, + "on_missing_parallelism": {"warning"}, + "on_disaggregated": {"warning"}, + "on_unknown_accelerator_count": {"warning"}, + "on_missing_window_extent": {"skip_count_comparison"}, + "on_missing_or_nonofficial": {"warning"}, + "fallback": {"whole_run_total"}, + "on_drift": {"warn_range_or_slope_required"}, + "disabled_warmup": {"exempt"}, + "on_missing_weights": {"arithmetic_mean"}, + "on_missing_required_dataset": {"error"}, + "on_unrelated_dataset": {"exclude"}, + "on_no_sample_counts": {"skip"}, + "on_unknown_model": {"warning"}, + "on_unknown_or_unpublished": {"warning"}, + "on_missing_metric": {"skip", "error"}, + "fraction_conversion": {"legacy_value_and_threshold_heuristic"}, + "count_scope": {"per_required_dataset_or_suite"}, + "units": {"samples_times_repeats"}, + "score": {"score_or_first_metric"}, + "unnamed_scalar": {"sole_metric_only_with_warning"}, + "on_multiple_results_per_band": {"average_and_warn"}, + "on_missing_bands": {"gate_available_mean_and_warn"}, + "on_no_results": {"warning"}, + } + for field, modes in allowed.items(): + value = check.rule.requirements.get(field) + if value is not None and value not in modes: + return unsupported(check, f"Unsupported policy mode {field}={value!r}") + return None diff --git a/src/inference_endpoint/validation/checks/metrics.py b/src/inference_endpoint/validation/checks/metrics.py new file mode 100644 index 000000000..39e88471b --- /dev/null +++ b/src/inference_endpoint/validation/checks/metrics.py @@ -0,0 +1,227 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy evaluators for metrics.""" + +from __future__ import annotations + +import math +import operator +from collections.abc import Mapping +from typing import Any + +from ..artifacts import Artifacts +from ..evaluator_base import Evaluator +from ..evidence.steady_state import SteadyStateStatus +from ..operations import ( + ComparisonOperator, + DerivedOperation, + DurationBasis, + OperandKind, +) +from ..outcomes import result +from ..planner import PlannedCheck +from ..results import CheckResult +from ..vocabulary import CheckKind + + +class Comparison(Evaluator, kind=CheckKind.COMPARISON): + operations = { + ComparisonOperator.GREATER_THAN: operator.gt, + ComparisonOperator.GREATER_THAN_OR_EQUAL: operator.ge, + ComparisonOperator.EQUAL: operator.eq, + } + + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + when = req.get("when") + if when: + actual = artifacts.resolve(when["field"], check) + if actual is None or actual <= when["greater_than"]: + return [] + + def operand(value: Mapping[str, Any]) -> Any: + return ( + value["value"] + if OperandKind(value["kind"]) is OperandKind.CONSTANT + else artifacts.resolve(value["field"], check) + ) + + left = operand(req["left"]) + right = operand(req["right"]) + op = self.operations.get(ComparisonOperator(req["operator"])) + if op is None: + return [ + result( + check, + False, + f"Unsupported comparison {req['operator']}", + blocked=True, + ) + ] + valid = ( + left is not None + and right is not None + and not isinstance(left, bool) + and not isinstance(right, bool) + and op(left, right) + ) + return [result(check, valid, f"{left!r} {req['operator']} {right!r}")] + + +class NumericValidity(Evaluator, kind=CheckKind.NUMERIC_VALIDITY): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + value = artifacts.resolve(req["source"], check) + if value is None and not req.get("require_present"): + return [] + valid = isinstance(value, (int, float)) and not isinstance(value, bool) + if valid and req.get("finite"): + valid = math.isfinite(value) + if valid and req.get("strictly_positive"): + valid = value > 0 + return [result(check, valid, f"{req['source']}: {value!r}")] + + +class Duration(Evaluator, kind=CheckKind.DURATION): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + point = artifacts.index.points[check.subject.id] + req = check.rule.requirements + band = artifacts.region(point) + if band is None and req.get("on_unclassifiable_concurrency") == "skip": + return [] + thresholds = artifacts.resolve(req["thresholds"], check) + minimum = thresholds.get(band) + duration = None + for basis in req["basis_precedence"]: + if basis == DurationBasis.REPORTED_WINDOW_ISSUE_SPAN: + value = point.get_config("steady_state.window.duration_s") + if value is not None and ( + not req.get("window_status_required") + or point.get_config("steady_state.status") + == SteadyStateStatus.WINDOWABLE + ): + duration = value * 1000 + elif basis == DurationBasis.WHOLE_RUN_DURATION: + value = point.get_summary("duration_ns") + if value is not None: + duration = value / 1e6 + if duration is not None: + break + if minimum is None: + return [ + result(check, False, "Duration threshold unavailable", blocked=True) + ] + return [ + result( + check, + isinstance(duration, (int, float)) + and math.isfinite(duration) + and duration >= minimum, + f"Duration {duration!r} ms; {band} requires {minimum} ms", + ) + ] + + +class DerivedMetric(Evaluator, kind=CheckKind.DERIVED_METRIC): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + operation = DerivedOperation(req["operation"]) + findings = [] + points = artifacts.members(check) + constants = artifacts.resolve(req.get("constants"), check) or {} + if operation is DerivedOperation.CURVE_PEAK_RATIO: + available = [ + p + for p in points + if p.throughput is not None + and isinstance(artifacts.resolve(req["stored"], check, p), (int, float)) + ] + peak = max( + (p.throughput for p in available if p.throughput is not None), default=0 + ) + if peak <= 0: + return [] + for point in available: + stored = artifacts.resolve(req["stored"], check, point) + assert point.throughput is not None + ratio = point.throughput / peak + findings.append( + result( + check, + math.isfinite(stored) + and abs(stored - ratio) <= req["tolerance"]["absolute"], + f"Utilization {stored}; derived {ratio}", + point=point, + ) + ) + return findings + for point in points: + stored = artifacts.resolve(req.get("stored"), check, point) + expected: float | None = None + if operation is DerivedOperation.OUTPUT_TOKENS_PER_ELAPSED_SECOND: + expected = point.throughput + elif operation is DerivedOperation.INVERSE_TPOT_P90_MS: + tpot = artifacts.resolve( + "result_summary.tpot.percentiles.90", check, point + ) + if isinstance(tpot, (int, float)) and math.isfinite(tpot) and tpot > 0: + expected = req["numerator"] / (tpot / 1e6) + else: + continue + elif operation is DerivedOperation.THROUGHPUT_PER_PROVISIONED_KW: + denominator = artifacts.resolve(req["denominator"], check, point) + if denominator is None or denominator <= 0: + continue + expected = ( + point.throughput / denominator + if point.throughput is not None + else None + ) + elif operation is DerivedOperation.OUTPUT_TOKENS_PER_COMPLETED_TURN_SECOND: + tokens = point.get_summary("output_tokens_per_turn_total") + seconds = point.get_summary("e2e_turn_time_seconds_total") + if ( + isinstance(tokens, (int, float)) + and isinstance(seconds, (int, float)) + and seconds > 0 + ): + expected = tokens / seconds + if expected is None and stored is None: + continue + else: + return [ + result( + check, + False, + f"Unsupported derived operation {operation}", + blocked=True, + ) + ] + if expected is None: + findings.append( + result(check, False, "Metric inputs unavailable", point=point) + ) + continue + if stored is None: + findings.append( + result(check, True, f"Derived metric {expected}", point=point) + ) + continue + floor = constants.get("relative_denominator_floor", 1e-9) + tolerance = constants.get("relative_tolerance", 0.01) + valid = ( + isinstance(stored, (int, float)) + and not isinstance(stored, bool) + and math.isfinite(stored) + and math.isfinite(expected) + and abs(stored - expected) / max(abs(expected), floor) <= tolerance + ) + findings.append( + result( + check, + valid, + f"Stored {stored!r}; derived {expected}; tolerance {tolerance}", + point=point, + ) + ) + return findings diff --git a/src/inference_endpoint/validation/checks/power.py b/src/inference_endpoint/validation/checks/power.py new file mode 100644 index 000000000..e3b5dd2c3 --- /dev/null +++ b/src/inference_endpoint/validation/checks/power.py @@ -0,0 +1,450 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy-driven power checks.""" + +from __future__ import annotations + +import math +import re +from collections.abc import Mapping +from dataclasses import replace +from pathlib import Path +from typing import Any, cast + +from pydantic import ValidationError + +from ..artifacts import Artifacts, PointArtifacts, read_mapping +from ..evaluator_base import Evaluator +from ..operations import PowerOperation +from ..planner import PlannedCheck, plan_checks +from ..power.calculation import PowerCalculator +from ..power.models import PowerComputation, PowerEvidence, SystemPower +from ..results import CheckResult +from ..types import Cooling, Decision, EvidenceKey, Scope +from ..vocabulary import CheckKind +from .helpers import finding, unsupported, validate_modes, warning + + +def power_entry(artifacts: Artifacts, point: PointArtifacts) -> PowerEvidence: + return artifacts.derived.power.get(str(point.path.parent.parent), {}) + + +def nodes_for( + point: PointArtifacts, computation: PowerComputation +) -> tuple[dict[int, int], list[str]]: + declared = point.get_config("nodes_used") + nodes = {} + problems = [] + if declared is not None: + for entry in declared: + ensemble, count = entry.get("system_node_ensemble_id"), entry.get("nodes") + candidates = [ + s for s in computation.sets if s.system_node_ensemble_id == ensemble + ] + if len(candidates) != 1 or ensemble in nodes: + problems.append(f"Unknown or duplicate node ensemble {ensemble}") + continue + if ( + not isinstance(count, int) + or isinstance(count, bool) + or not 1 <= count <= candidates[0].nodes_provisioned + ): + problems.append( + f"Invalid engaged node count {count!r} for ensemble {ensemble}" + ) + continue + nodes[ensemble] = count + return { + s.node_set_id: nodes.get(s.system_node_ensemble_id, s.nodes_provisioned) + for s in computation.sets + }, problems + + +def compute_descriptor(check: PlannedCheck, artifacts: Artifacts) -> None: + """Compute only the selected system using its effective descriptor rule.""" + system_id = check.subject.id + rule = check.rule + existing = artifacts.derived.power.get(system_id, {}) + if existing.get("requirements") == rule.requirements: + return + point_ids = artifacts.index.systems.get(system_id, ()) + for point_id in point_ids: + point = artifacts.index.points[point_id] + point.derived.power_kw = None + point.context = point.context.model_copy( + update={ + "available": point.context.available + - {EvidenceKey.POWER_COMPUTATION, EvidenceKey.POINT_POWER} + } + ) + constants = artifacts.resolve(rule.requirements["constants"], check) + path = Path(system_id) / rule.requirements["artifact"] + if not path.exists(): + artifacts.derived.power[system_id] = { + "problems": [f"Power descriptor {path.name} is missing"] + } + return + try: + descriptor = SystemPower.model_validate(read_mapping(path)) + members = [artifacts.index.points[id] for id in point_ids] + nodes: list[dict[str, Any]] = next( + (p.get_system("node_types", []) for p in members if p.system is not None), + [], + ) + cores = { + int(n["system_node_ensemble_id"]): n["host_processor_core_count"] + for n in nodes + if "system_node_ensemble_id" in n + and isinstance(n.get("host_processor_core_count"), int) + } + computation = PowerCalculator(descriptor, constants).compute(cores) + for node_set in computation.sets: + if node_set.accelerators_per_node is None: + node = next( + ( + n + for n in nodes + if str(n.get("system_node_ensemble_id")) + == str(node_set.system_node_ensemble_id) + ), + {}, + ) + counts = [ + a.get("accelerators_per_node") + for a in node.get("accelerator_info", ()) + if isinstance(a, Mapping) + ] + count = ( + sum(cast(int, count) for count in counts) + if counts and all(isinstance(c, int) for c in counts) + else node.get( + "accelerators_per_node", node.get("accelerator_count") + ) + ) + if isinstance(count, int): + computation.sets[computation.sets.index(node_set)] = replace( + node_set, accelerators_per_node=count + ) + if rule.requirements.get("cooling_must_match_system_description"): + declared = [str(p.get_system("cooling", "")) for p in members] + declared.extend(str(n.get("cooling", "")) for n in nodes) + labels: set[Cooling] = set() + for text in declared: + words = set(re.findall(r"[a-z]+", text.lower())) + if "mixed" in words: + labels.update((Cooling.AIR, Cooling.LIQUID)) + if words & {"liquid", "water", "immersion"}: + labels.add(Cooling.LIQUID) + if "air" in words: + labels.add(Cooling.AIR) + if labels and descriptor.cooling != ( + Cooling.MIXED if len(labels) > 1 else next(iter(labels)) + ): + computation.problems.append( + "Power cooling disagrees with system description" + ) + computation.provisioned_power_kw = None + entry: PowerEvidence = { + "computation": computation, + "problems": computation.problems, + "descriptor": descriptor, + "requirements": rule.requirements, + } + artifacts.derived.power[system_id] = entry + if computation.provisioned_power_kw is None: + return + for point in members: + point.context = point.context.model_copy( + update={ + "available": point.context.available + | {EvidenceKey.POWER_COMPUTATION} + } + ) + except (ValidationError, ValueError, KeyError, TypeError) as exc: + artifacts.derived.power[system_id] = { + "problems": [f"Power descriptor could not be computed: {exc}"] + } + + +def refresh_power_subjects(artifacts: Artifacts) -> None: + artifacts.index.subjects = [ + artifacts.index.points[subject.id].context + if subject.scope is Scope.POINT + else subject.model_copy( + update={ + "members": tuple( + artifacts.index.points[id].context + for id in artifacts.index.curves[subject.id] + ) + } + ) + if subject.scope is Scope.CURVE + else subject + for subject in artifacts.index.subjects + ] + + +def prepare(artifacts: Artifacts) -> None: + """Plan descriptor and point-power stages before publishing derived evidence.""" + artifacts.derived.power.clear() + for point in artifacts.index.points.values(): + point.derived.power_kw = None + point.context = point.context.model_copy( + update={ + "available": point.context.available + - {EvidenceKey.POWER_COMPUTATION, EvidenceKey.POINT_POWER} + } + ) + refresh_power_subjects(artifacts) + for scope, operation in ( + (Scope.SYSTEM, PowerOperation.DESCRIPTOR), + (Scope.POINT, PowerOperation.ENGAGED_NODES), + ): + subjects = [ + subject for subject in artifacts.index.subjects if subject.scope is scope + ] + if not subjects: + continue + stage = plan_checks(artifacts.policy, subjects) + for check in stage.checks: + if ( + check.decision is Decision.READY + and check.rule.kind is CheckKind.POWER + and check.rule.requirements.get("operation") == operation + ): + try: + PowerEvaluator()(check, artifacts) + except ( + ValueError, + TypeError, + KeyError, + ArithmeticError, + AttributeError, + ): + # The main evaluation emits the blocked finding. A failed + # prerequisite stage must not publish partial derived facts. + failed = {EvidenceKey.POINT_POWER} + if operation is PowerOperation.DESCRIPTOR: + artifacts.derived.power.pop(check.subject.id, None) + failed.add(EvidenceKey.POWER_COMPUTATION) + for point in artifacts.members(check): + point.derived.power_kw = None + point.context = point.context.model_copy( + update={"available": point.context.available - failed} + ) + refresh_power_subjects(artifacts) + + +class PowerEvaluator(Evaluator, kind=CheckKind.POWER): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + params = check.rule.requirements + invalid = validate_modes(check) + if invalid is not None: + return invalid + try: + operation = PowerOperation(cast(str, params.get("operation"))) + except ValueError: + return unsupported(check) + if operation is PowerOperation.DESCRIPTOR: + flags = ( + "validate_public_sources", + "validate_node_sets", + "validate_scale_out", + "validate_computed_values", + ) + if any(params.get(field) is False for field in flags): + return unsupported( + check, + "This power schema requires source, node, scale-out and computed-value validation", + ) + compute_descriptor(check, artifacts) + if operation in (PowerOperation.DESCRIPTOR, PowerOperation.APPENDIX_D_DEFAULTS): + entry = artifacts.derived.power.get(check.subject.id, {}) + computation = entry.get("computation") + if operation is PowerOperation.APPENDIX_D_DEFAULTS: + return ( + [ + warning( + check, + f"Estimated power uses defaults: {', '.join(computation.estimated)}", + ) + ] + if computation and computation.estimated + else [] + ) + problems = entry.get("problems", ["Power descriptor unavailable"]) + results = [ + finding( + check, + not problems, + "; ".join(problems) + or f"Provisioned power: {computation.provisioned_power_kw if computation else None} kW", + ) + ] + if computation: + results.extend( + warning(check, message) for message in computation.warnings + ) + return results + output = [] + for point in artifacts.members(check): + computation = power_entry(artifacts, point).get("computation") + if computation is None: + output.append( + finding( + check, + False, + "Power computation unavailable", + point=point, + blocked=True, + ) + ) + continue + engaged, problems = nodes_for(point, computation) + config_summary = point.get_system("config_summary") + description = { + **( + point.system.model_dump(mode="json", exclude_unset=True) + if point.system is not None + else {} + ), + **(config_summary if isinstance(config_summary, Mapping) else {}), + } + if operation is PowerOperation.ENGAGED_NODES: + point.derived.power_kw = None + point.context = point.context.model_copy( + update={ + "available": point.context.available - {EvidenceKey.POINT_POWER} + } + ) + if ( + params.get("declared_total_scaling") != "maximum_engaged_fraction" + or params.get("switch_scaling") != "engaged_node_fraction" + or params.get("on_undeclared_nodes") != "fully_engaged" + ): + return unsupported( + check, "Unsupported engaged power scaling policy" + ) + if not problems and computation.provisioned_power_kw is not None: + fractions = { + s.node_set_id: engaged[s.node_set_id] / s.nodes_provisioned + for s in computation.sets + } + if computation.declared: + kw = computation.provisioned_power_kw * max( + fractions.values(), default=1 + ) + else: + watts = sum( + s.power_w * fractions[s.node_set_id] + for s in computation.sets + ) + watts += ( + computation.scale_out_switch_power_w + * sum(engaged.values()) + / sum(s.nodes_provisioned for s in computation.sets) + ) + kw = watts / 1000 + point.derived.power_kw = round(kw, params["round_kw_decimals"]) + point.context = point.context.model_copy( + update={ + "available": point.context.available + | {EvidenceKey.POINT_POWER} + } + ) + output.append( + finding( + check, + point.derived.power_kw is not None, + f"Engaged power: {point.derived.power_kw} kW", + point=point, + ) + ) + continue + factors = params.get( + "replica_factors", + ("tensor_parallel", "pipeline_parallel", "expert_parallel"), + ) + present = any(field in description for field in (*factors, "data_parallel")) + replica = math.prod(description.get(field, 1) for field in factors) + dp = description.get("data_parallel", 1) + known = all(s.accelerators_per_node is not None for s in computation.sets) + if operation is PowerOperation.NODE_DECLARATION: + if ( + params.get("require_accelerator_capacity") + and present + and known + and not description.get("disaggregated") + ): + capacity = sum( + engaged[s.node_set_id] * cast(int, s.accelerators_per_node) + for s in computation.sets + ) + if capacity < replica * dp: + problems.append( + f"Declared nodes hold {capacity} accelerators; parallelism uses {replica * dp}" + ) + elif any( + engaged[s.node_set_id] > 0 + and capacity - cast(int, s.accelerators_per_node) + >= replica * dp + for s in computation.sets + ): + output.append( + warning( + check, + "Declared nodes include spare accelerator capacity", + point, + ) + ) + output.append( + finding( + check, + not problems, + "; ".join(problems) or "Engaged node declaration is valid", + point=point, + ) + ) + continue + if not present or description.get("disaggregated") or not known: + output.append( + warning( + check, + "Maximal engagement cannot be verified without shared parallelism and known accelerator counts", + point, + ) + ) + continue + provisioned = sum( + s.nodes_provisioned * cast(int, s.accelerators_per_node) + for s in computation.sets + ) + if replica < 1 or dp < 1: + output.append( + finding( + check, + False, + "Parallelism factors must be positive", + point=point, + ) + ) + continue + maximum = provisioned // replica + shortfall = artifacts.resolve(params["shortfall"], check, point) + valid = dp <= maximum + if params.get("require_maximum_data_parallel") and dp < maximum: + valid = ( + isinstance(shortfall, Mapping) + and shortfall.get("dp_actual") == dp + and shortfall.get("dp_formula") == maximum + and bool(shortfall.get("reason") or shortfall.get("explanation")) + ) + output.append( + finding( + check, + valid, + f"Data parallelism {dp}; maximum {maximum}; declared shortfall {shortfall!r}", + point=point, + ) + ) + return output diff --git a/src/inference_endpoint/validation/checks/seeds.py b/src/inference_endpoint/validation/checks/seeds.py new file mode 100644 index 000000000..203e0b8e3 --- /dev/null +++ b/src/inference_endpoint/validation/checks/seeds.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy-driven seeds checks.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import cast + +from ..artifacts import Artifacts, nested +from ..cohorts import Cohort, adoption_window +from ..evaluator_base import Evaluator +from ..operations import SeedOperation +from ..planner import PlannedCheck +from ..results import CheckResult +from ..vocabulary import CheckKind +from .helpers import finding, unsupported, validate_modes, warning + + +class SeedBindingEvaluator(Evaluator, kind=CheckKind.SEED_BINDING): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + params = check.rule.requirements + invalid = validate_modes(check) + if invalid is not None: + return invalid + try: + operation = SeedOperation(cast(str, params.get("operation"))) + except ValueError: + return unsupported(check) + registry = ( + artifacts.resolve(params.get("catalog"), check) + if operation is not SeedOperation.LEGACY_NAMES + else None + ) + if operation is not SeedOperation.LEGACY_NAMES and not isinstance( + registry, Mapping + ): + return unsupported(check, "Seed catalog is unavailable") + output = [] + for point in artifacts.members(check): + runtime = point.get_config("runtime_settings.runtime", {}) + if operation is SeedOperation.LEGACY_NAMES: + aliases = params["aliases"] + used = [ + key + for key in aliases + if f"runtime_settings.runtime.{key}" in point.evidence.config.fields + ] + if used and ( + not params.get("warn_legacy_only_when_model_seed_missing") + or runtime.get("model_seed") is None + ): + output.append( + warning( + check, f"Legacy seed fields used: {', '.join(used)}", point + ) + ) + continue + set_id = point.get_config("seed_set") + published = registry.get(set_id) + if operation is SeedOperation.MEMBERSHIP: + output.append( + finding( + check, + published is not None, + f"Seed set {set_id!r} {'is published' if published else 'is not published'}", + point=point, + ) + ) + continue + if published is None: + output.append( + finding( + check, + False, + f"Cannot evaluate unpublished seed set {set_id!r}", + point=point, + blocked=True, + ) + ) + continue + if operation is SeedOperation.RUNTIME_VALUES: + canonical = nested(point.config, "runtime_settings.runtime") + problems = [] + for field in params["fields"]: + expected = nested(published, field) + actual = ( + nested(canonical, field) + if canonical is not None + else runtime.get(field) + ) + if expected is None: + return unsupported(check, f"Published seed set has no {field}") + if actual != expected and ( + actual is not None or params.get("require_all") + ): + problems.append(f"{field}={actual!r}, expected {expected!r}") + output.append( + finding( + check, + not problems, + "; ".join(problems) + or "Runtime seed values match the published set", + point=point, + ) + ) + else: + cohorts = nested(published, "cohorts", ()) + if not cohorts: + output.append( + finding( + check, + False, + "Seed registry has no publication cohort; adoption cannot be verified", + point=point, + blocked=True, + ) + ) + continue + publication = Cohort.parse(min(cohorts)) + target = Cohort.parse(point.get_config("target_cohort") or "") + if publication is None or target is None: + output.append( + finding( + check, + False, + "Seed publication or target cohort cannot be parsed", + point=point, + blocked=True, + ) + ) + continue + allowed = adoption_window( + publication, params["adoption_window_cohorts"] + ) + output.append( + finding( + check, + target in allowed, + f"Seed adoption target {target}; allowed cohorts: {', '.join(map(str, allowed))}", + point=point, + ) + ) + return output diff --git a/src/inference_endpoint/validation/checks/steady_state.py b/src/inference_endpoint/validation/checks/steady_state.py new file mode 100644 index 000000000..fed99b0b2 --- /dev/null +++ b/src/inference_endpoint/validation/checks/steady_state.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy-driven steady state checks.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import cast + +from ..artifacts import Artifacts +from ..evaluator_base import Evaluator +from ..evidence.steady_state import MetricState, SteadyStateStatus +from ..operations import SteadyOperation +from ..planner import PlannedCheck +from ..results import CheckResult +from ..vocabulary import CheckKind +from .helpers import finding, number, unsupported, validate_modes, warning + + +class SteadyStateEvaluator(Evaluator, kind=CheckKind.STEADY_STATE): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + params = check.rule.requirements + invalid = validate_modes(check) + if invalid is not None: + return invalid + constants = artifacts.resolve(params["constants"], check) + if not isinstance(constants, Mapping): + return unsupported(check, "Steady-state catalog unavailable") + try: + operation = SteadyOperation(cast(str, params.get("operation"))) + except ValueError: + return unsupported(check) + output = [] + for point in artifacts.members(check): + block = point.config.steady_state if point.config is not None else None + if operation is SteadyOperation.REPORTING_BASIS: + if block is None or not block.is_official: + output.append( + warning( + check, + f"Steady-state window is nonofficial; reporting basis: {params['fallback']}", + point, + ) + ) + else: + output.append( + finding( + check, + True, + "Official steady-state window is the reporting basis", + point=point, + ) + ) + if block is not None and ( + block.verdict is not None + and block.verdict.is_drifting + or block.drifting_metrics + ): + output.append( + warning( + check, "Drifting metrics require a range or slope", point + ) + ) + continue + if block is None: + output.append( + finding( + check, + False, + "Steady-state declaration unavailable", + point=point, + blocked=True, + ) + ) + continue + problems = [] + if operation is SteadyOperation.VOCABULARY: + for field, catalog in (("status", "statuses"), ("verdict", "verdicts")): + if ( + getattr(block, field) is not None + and getattr(block, field) not in constants[catalog] + ): + problems.append( + f"{field}={getattr(block, field)!r} is outside the policy vocabulary" + ) + for metric, value in block.state.items(): + if value not in constants["metric_states"]: + problems.append(f"{metric}={value!r} is outside metric states") + else: + window = block.window + lo, hi = window.super_pass_start, window.super_pass_end + spans = hi - lo + 1 if number(lo) and number(hi) else None + minimum = constants["minimum_super_passes"] + if spans is not None and spans <= 0: + problems.append("Window super-pass extent is reversed or empty") + if block.status is SteadyStateStatus.WINDOWABLE: + if spans is not None and spans < minimum: + problems.append( + f"Official window spans {spans} super-passes; requires {minimum}" + ) + if params.get("official_requires_plateau") and any( + v is not MetricState.PLATEAU for v in block.state.values() + ): + problems.append("Official window contains a nonplateau metric") + elif ( + block.status is SteadyStateStatus.INSUFFICIENT_PASSES + and spans is not None + and spans >= minimum + ): + problems.append( + f"Insufficient-pass status conflicts with {spans} super-passes" + ) + reported = ( + window.n_super_passes + if "n_super_passes" in window.model_fields_set + else block.n_super_passes + ) + if ( + params.get("check_reported_super_pass_count") + and reported is not None + and spans is not None + and reported != spans + ): + problems.append( + f"Reported super-pass count {reported} differs from extent {spans}" + ) + output.append( + finding( + check, + not problems, + "; ".join(problems) + or "Steady-state declaration agrees with policy", + point=point, + ) + ) + return output diff --git a/src/inference_endpoint/validation/checks/structure.py b/src/inference_endpoint/validation/checks/structure.py new file mode 100644 index 000000000..154d5c91a --- /dev/null +++ b/src/inference_endpoint/validation/checks/structure.py @@ -0,0 +1,269 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy evaluators for structure.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from pathlib import Path +from typing import cast + +from inference_endpoint.config.schema import LoadPatternType + +from ..artifacts import Artifacts +from ..evaluator_base import Evaluator +from ..outcomes import result +from ..planner import PlannedCheck +from ..results import CheckResult, Severity +from ..vocabulary import CheckKind +from .helpers import values + + +class Presence(Evaluator, kind=CheckKind.PRESENCE): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + targets = req.get("targets") or [req.get("target")] + findings = [] + for target in targets: + value = ( + artifacts.root / target + if target in ("results", "docs", "src") + else artifacts.resolve(target, check) + ) + items = values(value) + if req.get("pattern"): + items = [ + item + for item in items + if re.fullmatch(req["pattern"], getattr(item, "name", str(item))) + ] + object_type = req.get("object") + + def present(item: object, object_type: str | None = object_type) -> bool: + if isinstance(item, Path): + return ( + item.is_file() + if object_type == "file" + else item.is_dir() + if object_type == "directory" + else item.exists() + ) + return item is not None and item is not False + + count = sum(present(item) for item in items) + findings.append( + result( + check, + count >= req.get("minimum_count", 1), + f"{target}: found {count}, required {req.get('minimum_count', 1)}", + ) + ) + return findings + + +class ArtifactSchema(Evaluator, kind=CheckKind.ARTIFACT_SCHEMA): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + point = artifacts.index.points[check.subject.id] + req = check.rule.requirements + schema = req.get("schema") + if schema == "accuracy_result": + present = point.evidence.accuracy.present + if req.get("when_present") and not present: + return [] + obj = point.accuracy + findings = point.evidence.accuracy.errors + if obj is not None and req.get("reject_empty") and not obj.root: + return [result(check, False, "Accuracy results must not be empty")] + else: + attr = { + "point_config": "config", + "system_description": "system", + "result_summary": "summary", + }.get(cast(str, schema)) + if attr is None: + return [ + result( + check, + False, + f"Unsupported artifact schema {schema}", + blocked=True, + ) + ] + obj = getattr(point, attr) + findings = getattr(point.evidence, attr).errors + if obj is None or any(f.severity == Severity.ERROR for f in findings): + return [ + f.model_copy( + update={ + "rule": check.rule.id, + "severity": Severity(check.rule.severity.value) + if f.severity is Severity.ERROR + else f.severity, + } + ) + for f in findings + ] or [result(check, False, f"{schema}: missing or invalid evidence")] + return [result(check, True, f"{schema}: parsed successfully")] + + +class Membership(Evaluator, kind=CheckKind.MEMBERSHIP): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + catalog = artifacts.resolve(req["catalog"], check) + items = values(artifacts.resolve(req["target"], check)) + findings = [] + seen = set() + for value in items: + if value is None and req.get("skip_missing"): + continue + if value is None and not req.get("required"): + continue + if req.get("deduplicate") and value in seen: + continue + seen.add(value) + findings.append( + result( + check, + value in catalog if catalog is not None else False, + f"{req['target']}: {value!r}; allowed {list(catalog or [])!r}", + blocked=catalog is None, + ) + ) + return findings + + +class Consistency(Evaluator, kind=CheckKind.CONSISTENCY): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + if "left" in req: + left = artifacts.resolve(req["left"], check) + right = artifacts.resolve(req["right"], check) + if left is None or right is None: + finding = result(check, False, "Comparison evidence is missing") + if req.get("on_missing") == "warning": + finding = finding.model_copy(update={"severity": Severity.WARNING}) + return [finding] + return [result(check, left == right, f"{left!r} compared with {right!r}")] + items = values(artifacts.resolve(req["target"], check)) + if req.get("exclude_fields"): + items = [ + {k: v for k, v in item.items() if k not in req["exclude_fields"]} + if isinstance(item, Mapping) + else item + for item in items + ] + if any(item is None for item in items): + if req.get("on_missing") == "error": + return [result(check, False, "Required comparison evidence is missing")] + items = [item for item in items if item is not None] + nonempty = ( + not req.get("require_nonempty") + or bool(items) + and all(bool(item) for item in items) + ) + return [ + result( + check, + nonempty and (not items or all(item == items[0] for item in items)), + f"{req['target']}: {len(items)} declarations compared", + ) + ] + + +class Disclosure(Evaluator, kind=CheckKind.DISCLOSURE): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + point = artifacts.index.points[check.subject.id] + fields = artifacts.resolve(req["fields"], check) + missing = [name for name in fields if point.get_config(name) in (None, "")] + return [ + result( + check, + not missing, + f"Missing disclosure fields: {missing}" + if missing + else "Required disclosures are present", + ) + ] + + +class CohortIdentifier(Evaluator, kind=CheckKind.COHORT_IDENTIFIER): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + value = artifacts.resolve(check.rule.requirements["target"], check) + return [ + result( + check, + isinstance(value, str) + and re.fullmatch(r"\d{4}-(0[1-9]|1[0-2])-C[01]", value) is not None, + f"Target cohort: {value!r}", + ) + ] + + +class PathResolution(Evaluator, kind=CheckKind.PATH_RESOLUTION): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + req = check.rule.requirements + root = Path(artifacts.resolve(req["base"], check)).resolve() + findings = [] + for address in req["targets"]: + value = artifacts.resolve(address, check) + if value is None and req.get("on_missing") == "skip": + continue + if not isinstance(value, str): + findings.append(result(check, False, f"{address}: missing path")) + continue + path = Path(value) + try: + resolved = (root / path).resolve() + except (OSError, RuntimeError) as error: + findings.append( + result( + check, False, f"{address}: cannot resolve {value!r}: {error}" + ) + ) + continue + valid = ( + (req.get("allow_absolute", False) or not path.is_absolute()) + and (req.get("allow_parent_traversal", False) or ".." not in path.parts) + and ( + req.get("allow_escape_via_symlink", False) + or resolved.is_relative_to(root) + ) + and ( + resolved.is_dir() + if req.get("require_directory") + else resolved.exists() + ) + ) + findings.append( + result(check, valid, f"{address}: {value!r} resolves to {resolved}") + ) + return findings + + +class ReportValue(Evaluator, kind=CheckKind.REPORT): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + value = artifacts.resolve(check.rule.requirements["target"], check) + return [result(check, True, f"{check.rule.requirements['target']}: {value!r}")] + + +class Issuance(Evaluator, kind=CheckKind.ISSUANCE): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + point = artifacts.index.points[check.subject.id] + req = check.rule.requirements + pattern = point.get_config( + "runtime_settings.load_pattern", LoadPatternType.CONCURRENCY + ) + valid = pattern in req["allowed"] and ( + not req.get("require_positive_concurrency") + or point.concurrency is not None + and point.concurrency > 0 + ) + return [ + result( + check, + valid, + f"Load pattern {pattern!r}, concurrency {point.concurrency}", + ) + ] diff --git a/src/inference_endpoint/validation/checks/warmup.py b/src/inference_endpoint/validation/checks/warmup.py new file mode 100644 index 000000000..b294b5b79 --- /dev/null +++ b/src/inference_endpoint/validation/checks/warmup.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy-driven warmup checks.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import cast + +from ..artifacts import Artifacts +from ..evaluator_base import Evaluator +from ..operations import WarmupOperation +from ..planner import PlannedCheck +from ..results import CheckResult +from ..vocabulary import CheckKind +from .helpers import block_value, finding, unsupported, validate_modes + + +class WarmupEvaluator(Evaluator, kind=CheckKind.WARMUP): + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + params = check.rule.requirements + invalid = validate_modes(check) + if invalid is not None: + return invalid + try: + operation = WarmupOperation(cast(str, params.get("operation"))) + except ValueError: + return unsupported(check) + output = [] + for point in artifacts.members(check): + if operation is WarmupOperation.SALT_DISABLED: + value = artifacts.resolve(params["source"], check, point) + if value is None and params.get("on_missing") == "skip": + continue + output.append( + finding( + check, + value is False, + f"Warmup salt disabled: {value!r}", + point=point, + ) + ) + continue + if params.get("verify_archive_contents"): + return unsupported( + check, "Warmup archive content verification is unavailable" + ) + warmup = block_value(point, "warmup") + if not isinstance(warmup, Mapping): + output.append( + finding( + check, + False, + "Warmup declaration unavailable", + point=point, + blocked=True, + ) + ) + continue + disabled = all( + warmup.get(key) == 0 + for key in ("duration_s", "requests_issued", "requests_completed") + ) + if disabled and params.get("disabled_warmup") == "exempt": + output.append( + finding( + check, + True, + "Disabled warmup is exempt from log retention", + point=point, + ) + ) + else: + retained = warmup.get("logs_retained") is True + output.append( + finding( + check, + retained or not params.get("require_declared_retained", True), + "Warmup logs retained" + if retained + else "Warmup log retention is not declared", + point=point, + ) + ) + return output diff --git a/src/inference_endpoint/validation/engine.py b/src/inference_endpoint/validation/engine.py new file mode 100644 index 000000000..75760f6b4 --- /dev/null +++ b/src/inference_endpoint/validation/engine.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Execute the selected cohort policy against classified submission evidence.""" + +from __future__ import annotations + +from os import PathLike +from pathlib import Path + +from . import bindings # noqa: F401 - register evaluator classes +from .artifacts import load_artifacts +from .checks import ( # noqa: F401 - register evaluator classes + collections, + datasets, + metrics, + structure, +) +from .checks.power import prepare +from .evaluator_base import Evaluator +from .models import Policy +from .outcomes import result +from .planner import plan_checks +from .results import Report +from .types import Decision + + +def execute_submission( + path: PathLike[str], + policy: Policy, +) -> Report: + """Parse evidence once, plan conditions/overrides, then execute each ready rule. + + Unavailable prerequisites and unsupported evaluators produce blocking errors. + Parsing does not add compliance findings outside the selected policy. + """ + artifacts = load_artifacts(Path(path), policy) + prepare(artifacts) + plan = plan_checks(policy, artifacts.index.subjects) + findings = [] + for check in plan.checks: + if check.decision is Decision.EXCLUDED: + continue + if check.decision is Decision.BLOCKED: + missing = ", ".join(sorted(check.missing)) + findings.append( + result( + check, + False, + check.reason + (f": {missing}" if missing else ""), + blocked=True, + ) + ) + continue + evaluator = Evaluator.lookup(check.rule.kind) + if evaluator is None: + findings.append( + result( + check, + False, + f"Unsupported evaluator kind: {check.rule.kind}", + blocked=True, + ) + ) + continue + try: + instance = evaluator() + findings.extend(instance(check, artifacts)) + except ( + ValueError, + TypeError, + KeyError, + ArithmeticError, + AttributeError, + ) as exc: + # Malformed evidence must never turn an unevaluated policy into success. + findings.append( + result( + check, False, f"Evaluation could not complete: {exc}", blocked=True + ) + ) + return Report(submission_path=artifacts.root, results=findings) diff --git a/src/inference_endpoint/validation/evaluator_base.py b/src/inference_endpoint/validation/evaluator_base.py new file mode 100644 index 000000000..d1bf43ab9 --- /dev/null +++ b/src/inference_endpoint/validation/evaluator_base.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Extensible callable-class dispatch for closed policy evaluator kinds.""" + +from abc import ABC, abstractmethod +from typing import ClassVar + +from .artifacts import Artifacts +from .planner import PlannedCheck +from .results import CheckResult +from .vocabulary import CheckKind + + +class Evaluator(ABC): + registry: ClassVar[dict[CheckKind, type["Evaluator"]]] = {} + + def __init_subclass__(cls, *, kind: CheckKind, **kwargs: object) -> None: + super().__init_subclass__(**kwargs) + if kind in cls.registry: + raise ValueError(f"Duplicate evaluator registration for {kind}") + cls.registry[kind] = cls + + @abstractmethod + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: + """Return findings from this rule's effective requirements.""" + raise NotImplementedError + + @classmethod + def lookup(cls, kind: CheckKind) -> type["Evaluator"] | None: + return cls.registry.get(kind) diff --git a/src/inference_endpoint/validation/outcomes.py b/src/inference_endpoint/validation/outcomes.py new file mode 100644 index 000000000..431093e68 --- /dev/null +++ b/src/inference_endpoint/validation/outcomes.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy finding construction shared by independent evaluator classes.""" + +from pathlib import Path + +from .artifacts import PointArtifacts +from .planner import PlannedCheck +from .results import CheckResult, Severity + + +def result( + check: PlannedCheck, + passed: bool, + message: str, + *, + key: str | None = None, + point: PointArtifacts | None = None, + blocked: bool = False, +) -> CheckResult: + return CheckResult( + rule=check.rule.id, + key=key or ("blocked" if blocked else "pass" if passed else "fail"), + title=check.rule.id, + message=message, + severity=Severity.INFO + if passed + else Severity.ERROR + if blocked + else Severity(check.rule.severity.value), + path=point.path if point else Path(check.subject.id), + ) diff --git a/tests/fixtures/validation/submissions/invalid_submission/docs/.gitkeep b/tests/fixtures/validation/submissions/invalid_submission/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/accuracy_results.json b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/accuracy_results.json new file mode 100644 index 000000000..49247cf54 --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/accuracy_results.json @@ -0,0 +1,13 @@ +{ + "cnn_dailymail::llama3_8b": { + "dataset_name": "cnn_dailymail::llama3_8b", + "num_samples": 500, + "extractor": "RougeExtractor", + "score": { + "rouge1": "30.00", + "rouge2": "12.00", + "rougeL": "20.00" + }, + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/config.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/config.yaml new file mode 100644 index 000000000..f9afe658a --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/config.yaml @@ -0,0 +1 @@ +concurrency: 16 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/point.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/point.yaml new file mode 100644 index 000000000..cdfb6a1db --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/point.yaml @@ -0,0 +1,49 @@ +concurrency: 16 +dataset: llm-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 88 +model_name: Llama-3_1-8B-Instruct +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 16 + data_source: llm-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 160 + requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/result_summary.json b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/result_summary.json new file mode 100644 index 000000000..e6e2bc2a1 --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 3000, + "n_samples_completed": 3000, + "n_samples_failed": 0, + "duration_ns": 600000000000, + "ttft": { + "total": 135600000000.0, + "percentiles": { + "50": 45200000.0, + "95": 89100000.0, + "90": 80320000.0 + } + }, + "output_sequence_lengths": { + "total": 480000.0, + "percentiles": {} + }, + "tpot": { + "total": 12147000000.000002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 800.0, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/system_desc.json b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/system_desc.json new file mode 100644 index 000000000..2ba326397 --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Invalid Corp", + "submitter_contact": "contact@example.com", + "system_name": "bad_system", + "system_category": "datacenter", + "max_supported_concurrency": 88, + "serving_framework": "custom 0.0.1", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Generic CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 8, + "host_memory_capacity": "64 GB", + "host_networking": "Ethernet 1Gb", + "host_storage_capacity": "1 TB", + "host_storage_type": "HDD", + "operating_system": "Ubuntu 20.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Generic GPU", + "accelerators_per_node": 1, + "accelerator_memory_capacity": "16 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "Llama-3_1-8B-Instruct", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 1, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.32 +} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/accuracy_results.json b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/accuracy_results.json new file mode 100644 index 000000000..49247cf54 --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/accuracy_results.json @@ -0,0 +1,13 @@ +{ + "cnn_dailymail::llama3_8b": { + "dataset_name": "cnn_dailymail::llama3_8b", + "num_samples": 500, + "extractor": "RougeExtractor", + "score": { + "rouge1": "30.00", + "rouge2": "12.00", + "rougeL": "20.00" + }, + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/config.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/config.yaml new file mode 100644 index 000000000..1a6f2cb8e --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/config.yaml @@ -0,0 +1 @@ +concurrency: 38 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/point.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/point.yaml new file mode 100644 index 000000000..821700459 --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/point.yaml @@ -0,0 +1,49 @@ +concurrency: 38 +dataset: llm-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 88 +model_name: Llama-3_1-8B-Instruct +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 38 + data_source: llm-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 380 + requests_issued: 380 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/result_summary.json b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/result_summary.json new file mode 100644 index 000000000..0fe18b542 --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 8000, + "n_samples_completed": 8000, + "n_samples_failed": 0, + "duration_ns": 1200000000000, + "ttft": { + "total": 361600000000.0, + "percentiles": { + "50": 45200000.0, + "95": 89100000.0, + "90": 80320000.0 + } + }, + "output_sequence_lengths": { + "total": 1800000.0, + "percentiles": {} + }, + "tpot": { + "total": 32392000000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1500.0, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/system_desc.json b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/system_desc.json new file mode 100644 index 000000000..7e874b293 --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Invalid Corp", + "submitter_contact": "contact@example.com", + "system_name": "bad_system", + "system_category": "datacenter", + "max_supported_concurrency": 88, + "serving_framework": "custom 0.0.1", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Generic CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 8, + "host_memory_capacity": "64 GB", + "host_networking": "Ethernet 1Gb", + "host_storage_capacity": "1 TB", + "host_storage_type": "HDD", + "operating_system": "Ubuntu 20.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Generic GPU", + "accelerators_per_node": 1, + "accelerator_memory_capacity": "16 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "Llama-3_1-8B-Instruct", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 1, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.6 +} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/accuracy_results.json b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/accuracy_results.json new file mode 100644 index 000000000..49247cf54 --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/accuracy_results.json @@ -0,0 +1,13 @@ +{ + "cnn_dailymail::llama3_8b": { + "dataset_name": "cnn_dailymail::llama3_8b", + "num_samples": 500, + "extractor": "RougeExtractor", + "score": { + "rouge1": "30.00", + "rouge2": "12.00", + "rougeL": "20.00" + }, + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/config.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/config.yaml new file mode 100644 index 000000000..cf16abd9b --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/config.yaml @@ -0,0 +1 @@ +concurrency: 88 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/point.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/point.yaml new file mode 100644 index 000000000..42a5783f8 --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/point.yaml @@ -0,0 +1,50 @@ +concurrency: 88 +dataset: llm-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 88 +model_name: Llama-3_1-8B-Instruct +model_notes: "" +model_precision: FP16 +offline: elected +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 88 + data_source: llm-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 880 + requests_issued: 880 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/result_summary.json b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/result_summary.json new file mode 100644 index 000000000..839c42f09 --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 5000, + "n_samples_completed": 5000, + "n_samples_failed": 0, + "duration_ns": 1200000000000, + "ttft": { + "total": 226000000000.0, + "percentiles": { + "50": 45200000.0, + "95": 89100000.0, + "90": 80320000.0 + } + }, + "output_sequence_lengths": { + "total": 3000000.0, + "percentiles": {} + }, + "tpot": { + "total": 20245000000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 2500.0, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/system_desc.json b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/system_desc.json new file mode 100644 index 000000000..980e8279b --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Invalid Corp", + "submitter_contact": "contact@example.com", + "system_name": "bad_system", + "system_category": "datacenter", + "max_supported_concurrency": 88, + "serving_framework": "custom 0.0.1", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Generic CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 8, + "host_memory_capacity": "64 GB", + "host_networking": "Ethernet 1Gb", + "host_storage_capacity": "1 TB", + "host_storage_type": "HDD", + "operating_system": "Ubuntu 20.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Generic GPU", + "accelerators_per_node": 1, + "accelerator_memory_capacity": "16 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "Llama-3_1-8B-Instruct", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 1, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 1.0 +} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/system_power.json b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/system_power.json new file mode 100644 index 000000000..ef7d5ad07 --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/system_power.json @@ -0,0 +1,44 @@ +{ + "system_desc_id": "bad_system", + "cooling": "air", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": { + "model": "Generic CPU", + "count_per_node": 1, + "tdp_per_unit": { + "value_w": 350, + "source_type": "vendor_spec", + "source": "https://example.com/cpu-spec" + } + }, + "accelerator": { + "model": "Generic GPU", + "count_per_node": 1, + "tdp_per_unit": { + "value_w": 700, + "source_type": "vendor_spec", + "source": "https://example.com/accelerator-spec" + } + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": { + "value_w": 3500, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + } + } + ], + "scale_out": { + "present": false + } +} diff --git a/tests/fixtures/validation/submissions/invalid_submission/src/custom/README.md b/tests/fixtures/validation/submissions/invalid_submission/src/custom/README.md new file mode 100644 index 000000000..25fbd778e --- /dev/null +++ b/tests/fixtures/validation/submissions/invalid_submission/src/custom/README.md @@ -0,0 +1,3 @@ +# custom + +Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_a/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_a/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/config.yaml new file mode 100644 index 000000000..b8581faa1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/config.yaml @@ -0,0 +1 @@ +concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/point.yaml new file mode 100644 index 000000000..092fd8959 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/point.yaml @@ -0,0 +1,49 @@ +concurrency: 1024 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1024 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 10240 + requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/result_summary.json new file mode 100644 index 000000000..07c4aa73c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 2302757311892.0, + "ttft": { + "total": 551183759784528.0, + "percentiles": { + "50": 86176322668.0, + "95": 100733262106.25, + "90": 97821874218.6 + } + }, + "output_sequence_lengths": { + "total": 22417180.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 9734.9295, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/system_desc.json new file mode 100644 index 000000000..f3d7279df --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-A", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.997637 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/config.yaml new file mode 100644 index 000000000..51457c3cf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/config.yaml @@ -0,0 +1 @@ +concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/point.yaml new file mode 100644 index 000000000..8c27c4dd8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/point.yaml @@ -0,0 +1,49 @@ +concurrency: 128 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 128 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 1280 + requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/result_summary.json new file mode 100644 index 000000000..d04481dea --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 3940004920032.0, + "ttft": { + "total": 1084206049836.0, + "percentiles": { + "50": 169513141.0, + "95": 261888051.50000003, + "90": 243413069.40000004 + } + }, + "output_sequence_lengths": { + "total": 22407098.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 5687.0736, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/system_desc.json new file mode 100644 index 000000000..1abf0179c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-A", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.582812 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/config.yaml new file mode 100644 index 000000000..f9afe658a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/config.yaml @@ -0,0 +1 @@ +concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/point.yaml new file mode 100644 index 000000000..7864405d8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/point.yaml @@ -0,0 +1,49 @@ +concurrency: 16 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 16 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 160 + requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/result_summary.json new file mode 100644 index 000000000..41eff27bd --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 11856610466381.0, + "ttft": { + "total": 599489491146.0, + "percentiles": { + "50": 93728813.5, + "95": 145818384.25, + "90": 135400470.1 + } + }, + "output_sequence_lengths": { + "total": 22387661.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1888.2008, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/system_desc.json new file mode 100644 index 000000000..8a0a841bc --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-A", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.193503 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/config.yaml new file mode 100644 index 000000000..6b53d1bba --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/config.yaml @@ -0,0 +1 @@ +concurrency: 2048 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/point.yaml new file mode 100644 index 000000000..29199e39a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/point.yaml @@ -0,0 +1,50 @@ +concurrency: 2048 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +offline: elected +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 2048 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 20480 + requests_issued: 20480 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/result_summary.json new file mode 100644 index 000000000..db91311d5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 2294246323608.9995, + "ttft": { + "total": 2845856057894070.0, + "percentiles": { + "50": 444943098482.5, + "95": 462411343300.0, + "90": 458917694336.5 + } + }, + "output_sequence_lengths": { + "total": 22387231.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 9757.9893, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/system_desc.json new file mode 100644 index 000000000..65992fcd6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-A", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 1.0 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/config.yaml new file mode 100644 index 000000000..24d4dedbd --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/config.yaml @@ -0,0 +1 @@ +concurrency: 4 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/point.yaml new file mode 100644 index 000000000..1a6d3dfe2 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/point.yaml @@ -0,0 +1,49 @@ +concurrency: 4 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 4 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 40 + requests_issued: 40 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/result_summary.json new file mode 100644 index 000000000..32bec22e7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 28913949735508.0, + "ttft": { + "total": 521833969800.0, + "percentiles": { + "50": 81587550.0, + "95": 183622220.0, + "90": 163215286.0 + } + }, + "output_sequence_lengths": { + "total": 22427551.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 775.6654, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/system_desc.json new file mode 100644 index 000000000..5e287b0e8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-A", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.07949 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/config.yaml new file mode 100644 index 000000000..846832e5e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/config.yaml @@ -0,0 +1 @@ +concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/point.yaml new file mode 100644 index 000000000..a3b2de6dd --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/point.yaml @@ -0,0 +1,49 @@ +concurrency: 512 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 512 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 5120 + requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/result_summary.json new file mode 100644 index 000000000..2fb9226cc --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 2427693763211.0, + "ttft": { + "total": 2446935638394.0, + "percentiles": { + "50": 382572801.5, + "95": 10778207187.0, + "90": 8699080309.900002 + } + }, + "output_sequence_lengths": { + "total": 22315593.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 9192.0955, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/system_desc.json new file mode 100644 index 000000000..994bd5df1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-A", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.942007 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/config.yaml new file mode 100644 index 000000000..7db6216f5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/config.yaml @@ -0,0 +1 @@ +concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/point.yaml new file mode 100644 index 000000000..c57905824 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/point.yaml @@ -0,0 +1,49 @@ +concurrency: 64 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 64 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 640 + requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/result_summary.json new file mode 100644 index 000000000..f69318dbf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 5163749763587.001, + "ttft": { + "total": 814271667534.0, + "percentiles": { + "50": 127309516.5, + "95": 193387620.0, + "90": 180171999.3 + } + }, + "output_sequence_lengths": { + "total": 22246057.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 4308.1207, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/system_desc.json new file mode 100644 index 000000000..0b97f12f6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-A", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.441497 +} diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/system_power.json b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/system_power.json new file mode 100644 index 000000000..5d5bb073d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/system_power.json @@ -0,0 +1,44 @@ +{ + "system_desc_id": "sys_mi355x_8gpu", + "cooling": "air", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": { + "model": "AMD EPYC 9654", + "count_per_node": 2, + "tdp_per_unit": { + "value_w": 350, + "source_type": "vendor_spec", + "source": "https://example.com/cpu-spec" + } + }, + "accelerator": { + "model": "AMD Instinct MI355X", + "count_per_node": 8, + "tdp_per_unit": { + "value_w": 700, + "source_type": "vendor_spec", + "source": "https://example.com/accelerator-spec" + } + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": { + "value_w": 3500, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + } + } + ], + "scale_out": { + "present": false + } +} diff --git a/tests/fixtures/validation/submissions/sub_a/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_a/src/vllm/README.md new file mode 100644 index 000000000..be3edfb30 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_a/src/vllm/README.md @@ -0,0 +1,3 @@ +# vllm + +Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_b/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_b/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/config.yaml new file mode 100644 index 000000000..b8581faa1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/config.yaml @@ -0,0 +1 @@ +concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/point.yaml new file mode 100644 index 000000000..092fd8959 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/point.yaml @@ -0,0 +1,49 @@ +concurrency: 1024 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1024 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 10240 + requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/result_summary.json new file mode 100644 index 000000000..ef0d7d52f --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 2555878737455.0, + "ttft": { + "total": 607712738731644.0, + "percentiles": { + "50": 95014499489.0, + "95": 113055413992.5, + "90": 109447231091.8 + } + }, + "output_sequence_lengths": { + "total": 22287872.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 8720.2384, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/system_desc.json new file mode 100644 index 000000000..a4963a2ea --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-B", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_16gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 16, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 16, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 1.0 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/config.yaml new file mode 100644 index 000000000..51457c3cf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/config.yaml @@ -0,0 +1 @@ +concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/point.yaml new file mode 100644 index 000000000..8c27c4dd8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/point.yaml @@ -0,0 +1,49 @@ +concurrency: 128 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 128 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 1280 + requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/result_summary.json new file mode 100644 index 000000000..c10338bc1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 4564665788130.0, + "ttft": { + "total": 1276608945222.0, + "percentiles": { + "50": 199594894.5, + "95": 306682164.0, + "90": 285264710.1 + } + }, + "output_sequence_lengths": { + "total": 22291611.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 4883.5144, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/system_desc.json new file mode 100644 index 000000000..8972b6334 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-B", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_16gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 16, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 16, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.560021 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/config.yaml new file mode 100644 index 000000000..f9afe658a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/config.yaml @@ -0,0 +1 @@ +concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/point.yaml new file mode 100644 index 000000000..7864405d8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/point.yaml @@ -0,0 +1,49 @@ +concurrency: 16 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 16 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 160 + requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/result_summary.json new file mode 100644 index 000000000..e6d7a27c2 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 13160683481153.0, + "ttft": { + "total": 707502001908.0, + "percentiles": { + "50": 110616323.0, + "95": 220638667.75, + "90": 198634198.8 + } + }, + "output_sequence_lengths": { + "total": 22403343.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1702.2933, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/system_desc.json new file mode 100644 index 000000000..41d2cef5c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-B", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_16gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 16, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 16, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.195212 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/config.yaml new file mode 100644 index 000000000..6b53d1bba --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/config.yaml @@ -0,0 +1 @@ +concurrency: 2048 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/point.yaml new file mode 100644 index 000000000..29199e39a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/point.yaml @@ -0,0 +1,50 @@ +concurrency: 2048 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +offline: elected +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 2048 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 20480 + requests_issued: 20480 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/result_summary.json new file mode 100644 index 000000000..7db6a29de --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 2587004792635.0, + "ttft": { + "total": 3198282303472386.0, + "percentiles": { + "50": 500044137503.5, + "95": 545254932484.0, + "90": 536212773487.9 + } + }, + "output_sequence_lengths": { + "total": 22359894.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 8643.1591, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/system_desc.json new file mode 100644 index 000000000..3a0110165 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-B", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_16gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 16, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 16, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.991161 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/config.yaml new file mode 100644 index 000000000..24d4dedbd --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/config.yaml @@ -0,0 +1 @@ +concurrency: 4 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/point.yaml new file mode 100644 index 000000000..1a6d3dfe2 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/point.yaml @@ -0,0 +1,49 @@ +concurrency: 4 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 4 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 40 + requests_issued: 40 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/result_summary.json new file mode 100644 index 000000000..a8be486a4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 30408127168980.996, + "ttft": { + "total": 562383145116.0, + "percentiles": { + "50": 87927321.0, + "95": 197007595.0, + "90": 175191540.2 + } + }, + "output_sequence_lengths": { + "total": 22330923.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 734.3735, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/system_desc.json new file mode 100644 index 000000000..cc430f766 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-B", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_16gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 16, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 16, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.084215 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/config.yaml new file mode 100644 index 000000000..846832e5e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/config.yaml @@ -0,0 +1 @@ +concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/point.yaml new file mode 100644 index 000000000..a3b2de6dd --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/point.yaml @@ -0,0 +1,49 @@ +concurrency: 512 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 512 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 5120 + requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/result_summary.json new file mode 100644 index 000000000..62292160c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 2755067877868.0, + "ttft": { + "total": 2813058879462.0, + "percentiles": { + "50": 439815334.5, + "95": 11423621862.5, + "90": 9226860556.9 + } + }, + "output_sequence_lengths": { + "total": 22400639.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 8130.7031, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/system_desc.json new file mode 100644 index 000000000..620a0a0b8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-B", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_16gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 16, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 16, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.932395 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/accuracy_results.json new file mode 100644 index 000000000..b59ac0f79 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 4395, + "score": { + "exact_match": 83.13 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/config.yaml new file mode 100644 index 000000000..7db6216f5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/config.yaml @@ -0,0 +1 @@ +concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/point.yaml new file mode 100644 index 000000000..c57905824 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/point.yaml @@ -0,0 +1,49 @@ +concurrency: 64 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: gpt-oss-120b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 64 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 640 + requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/result_summary.json new file mode 100644 index 000000000..c26fef001 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 6396, + "n_samples_completed": 6396, + "n_samples_failed": 0, + "duration_ns": 6369959177476.999, + "ttft": { + "total": 994203277548.0, + "percentiles": { + "50": 155441413.0, + "95": 237329605.75, + "90": 220951967.2 + } + }, + "output_sequence_lengths": { + "total": 22395866.0, + "percentiles": {} + }, + "tpot": { + "total": 25897404000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 3515.8571, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/system_desc.json new file mode 100644 index 000000000..a34829f6d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-B", + "submitter_contact": "contact@example.com", + "system_name": "sys_mi355x_16gpu", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "AMD Instinct MI355X", + "accelerators_per_node": 16, + "accelerator_memory_capacity": "288 GB", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "gpt-oss-120b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 16, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.403184 +} diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/system_power.json b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/system_power.json new file mode 100644 index 000000000..79ae07df0 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/system_power.json @@ -0,0 +1,44 @@ +{ + "system_desc_id": "sys_mi355x_16gpu", + "cooling": "air", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": { + "model": "AMD EPYC 9654", + "count_per_node": 2, + "tdp_per_unit": { + "value_w": 350, + "source_type": "vendor_spec", + "source": "https://example.com/cpu-spec" + } + }, + "accelerator": { + "model": "AMD Instinct MI355X", + "count_per_node": 16, + "tdp_per_unit": { + "value_w": 700, + "source_type": "vendor_spec", + "source": "https://example.com/accelerator-spec" + } + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": { + "value_w": 3500, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + } + } + ], + "scale_out": { + "present": false + } +} diff --git a/tests/fixtures/validation/submissions/sub_b/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_b/src/vllm/README.md new file mode 100644 index 000000000..be3edfb30 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_b/src/vllm/README.md @@ -0,0 +1,3 @@ +# vllm + +Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_c/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_c/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/config.yaml new file mode 100644 index 000000000..51457c3cf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/config.yaml @@ -0,0 +1 @@ +concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/point.yaml new file mode 100644 index 000000000..cd5f4fc95 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/point.yaml @@ -0,0 +1,49 @@ +concurrency: 128 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 128 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 1280 + requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/result_summary.json new file mode 100644 index 000000000..fc8a189c0 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 3704248419955.0, + "ttft": { + "total": 5674433556480.0, + "percentiles": { + "50": 230893292.5, + "95": 330418109.5, + "90": 310513146.1 + } + }, + "output_sequence_lengths": { + "total": 7189576.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1940.9001, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/system_desc.json new file mode 100644 index 000000000..260d1aeeb --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-C", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_4chip", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 4, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 4, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.686722 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/config.yaml new file mode 100644 index 000000000..f9afe658a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/config.yaml @@ -0,0 +1 @@ +concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/point.yaml new file mode 100644 index 000000000..49f86ecbf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/point.yaml @@ -0,0 +1,49 @@ +concurrency: 16 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 16 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 160 + requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/result_summary.json new file mode 100644 index 000000000..13e0192b4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 9681657879146.0, + "ttft": { + "total": 2364827467776.0, + "percentiles": { + "50": 96225076.0, + "95": 164579607.0, + "90": 150908700.8 + } + }, + "output_sequence_lengths": { + "total": 7193577.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 743.0109, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/system_desc.json new file mode 100644 index 000000000..f646966d8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-C", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_4chip", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 4, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 4, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.262889 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/config.yaml new file mode 100644 index 000000000..f928653b5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/config.yaml @@ -0,0 +1 @@ +concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/point.yaml new file mode 100644 index 000000000..71e2cd93e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/point.yaml @@ -0,0 +1,49 @@ +concurrency: 256 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 256 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 2560 + requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/result_summary.json new file mode 100644 index 000000000..e49af2020 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 3055408461497.0005, + "ttft": { + "total": 8782727024640.0, + "percentiles": { + "50": 357370077.5, + "95": 457645674.25, + "90": 437590554.9 + } + }, + "output_sequence_lengths": { + "total": 7178996.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 2349.6027, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/system_desc.json new file mode 100644 index 000000000..bd8840a07 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-C", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_4chip", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 4, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 4, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.831327 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/config.yaml new file mode 100644 index 000000000..b03f20973 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/config.yaml @@ -0,0 +1 @@ +concurrency: 32 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/point.yaml new file mode 100644 index 000000000..81faebf4f --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/point.yaml @@ -0,0 +1,49 @@ +concurrency: 32 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 32 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 320 + requests_issued: 320 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/result_summary.json new file mode 100644 index 000000000..a53ebf6c8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 7215663280145.0, + "ttft": { + "total": 3557367164928.0, + "percentiles": { + "50": 144749640.5, + "95": 211749072.25, + "90": 198349185.9 + } + }, + "output_sequence_lengths": { + "total": 7193551.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 996.9355, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/system_desc.json new file mode 100644 index 000000000..dc4e3fb6d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-C", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_4chip", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 4, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 4, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.352732 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/config.yaml new file mode 100644 index 000000000..846832e5e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/config.yaml @@ -0,0 +1 @@ +concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/point.yaml new file mode 100644 index 000000000..bc041691d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/point.yaml @@ -0,0 +1,50 @@ +concurrency: 512 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +offline: elected +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 512 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 5120 + requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/result_summary.json new file mode 100644 index 000000000..d3f1c582f --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 2550450936890.0, + "ttft": { + "total": 13680033595392.0, + "percentiles": { + "50": 556641992.0, + "95": 841800473.5, + "90": 784768777.2 + } + }, + "output_sequence_lengths": { + "total": 7208408.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 2826.3269, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/system_desc.json new file mode 100644 index 000000000..ab52c2334 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-C", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_4chip", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 4, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 4, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 1.0 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/config.yaml new file mode 100644 index 000000000..7db6216f5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/config.yaml @@ -0,0 +1 @@ +concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/point.yaml new file mode 100644 index 000000000..f7a332096 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/point.yaml @@ -0,0 +1,49 @@ +concurrency: 64 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 64 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 640 + requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/result_summary.json new file mode 100644 index 000000000..d7c5651f6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 4947046368648.0, + "ttft": { + "total": 3985956200448.0, + "percentiles": { + "50": 162188973.0, + "95": 255115666.5, + "90": 236530327.8 + } + }, + "output_sequence_lengths": { + "total": 7194216.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1454.2447, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/system_desc.json new file mode 100644 index 000000000..42152ddcc --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-C", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_4chip", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 4, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 4, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.514535 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/config.yaml new file mode 100644 index 000000000..e1b56a7e8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/config.yaml @@ -0,0 +1 @@ +concurrency: 8 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/point.yaml new file mode 100644 index 000000000..f195dbf92 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/point.yaml @@ -0,0 +1,49 @@ +concurrency: 8 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 8 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 80 + requests_issued: 80 diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/result_summary.json new file mode 100644 index 000000000..35c0965bb --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 17117165266230.002, + "ttft": { + "total": 2276072361984.0, + "percentiles": { + "50": 92613621.5, + "95": 157459882.0, + "90": 144490629.9 + } + }, + "output_sequence_lengths": { + "total": 7185824.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 419.8022, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/system_desc.json new file mode 100644 index 000000000..492bd9fd7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-C", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_4chip", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 4, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 4, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.148533 +} diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/system_power.json b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/system_power.json new file mode 100644 index 000000000..084553bb8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/system_power.json @@ -0,0 +1,44 @@ +{ + "system_desc_id": "sys_tpu_4chip", + "cooling": "liquid", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": { + "model": "Custom TPU Host CPU", + "count_per_node": 1, + "tdp_per_unit": { + "value_w": 350, + "source_type": "vendor_spec", + "source": "https://example.com/cpu-spec" + } + }, + "accelerator": { + "model": "Google TPU v5p", + "count_per_node": 4, + "tdp_per_unit": { + "value_w": 700, + "source_type": "vendor_spec", + "source": "https://example.com/accelerator-spec" + } + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": { + "value_w": 3500, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + } + } + ], + "scale_out": { + "present": false + } +} diff --git a/tests/fixtures/validation/submissions/sub_c/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_c/src/vllm/README.md new file mode 100644 index 000000000..be3edfb30 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_c/src/vllm/README.md @@ -0,0 +1,3 @@ +# vllm + +Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_d/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_d/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/config.yaml new file mode 100644 index 000000000..b8581faa1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/config.yaml @@ -0,0 +1 @@ +concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/point.yaml new file mode 100644 index 000000000..9f37e80ab --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/point.yaml @@ -0,0 +1,50 @@ +concurrency: 1024 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +offline: elected +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1024 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 10240 + requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/result_summary.json new file mode 100644 index 000000000..d9903b6b2 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 2448212914685.0, + "ttft": { + "total": 19499568439296.0, + "percentiles": { + "50": 793439471.0, + "95": 1030084586.4999999, + "90": 982755563.3999999 + } + }, + "output_sequence_lengths": { + "total": 7186180.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 2935.2758, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/system_desc.json new file mode 100644 index 000000000..9d3e336e4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-D", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_8chip", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 1.0 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/config.yaml new file mode 100644 index 000000000..51457c3cf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/config.yaml @@ -0,0 +1 @@ +concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/point.yaml new file mode 100644 index 000000000..08573ee57 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/point.yaml @@ -0,0 +1,49 @@ +concurrency: 128 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 128 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 1280 + requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/result_summary.json new file mode 100644 index 000000000..6d128c4d2 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 4981652524410.999, + "ttft": { + "total": 5200658755584.0, + "percentiles": { + "50": 211615346.5, + "95": 274691318.25, + "90": 262076123.9 + } + }, + "output_sequence_lengths": { + "total": 7208951.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1447.1003, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/system_desc.json new file mode 100644 index 000000000..872271340 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-D", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_8chip", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.493003 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/config.yaml new file mode 100644 index 000000000..f9afe658a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/config.yaml @@ -0,0 +1 @@ +concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/point.yaml new file mode 100644 index 000000000..afe70bf5d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/point.yaml @@ -0,0 +1,49 @@ +concurrency: 16 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 16 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 160 + requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/result_summary.json new file mode 100644 index 000000000..9ba4737f5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 21458438875063.0, + "ttft": { + "total": 3068488581120.0, + "percentiles": { + "50": 124857120.0, + "95": 181258587.5, + "90": 169978294.0 + } + }, + "output_sequence_lengths": { + "total": 7214114.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 336.1901, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/system_desc.json new file mode 100644 index 000000000..e7d21f342 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-D", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_8chip", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.114534 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/config.yaml new file mode 100644 index 000000000..f928653b5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/config.yaml @@ -0,0 +1 @@ +concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/point.yaml new file mode 100644 index 000000000..9295c1999 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/point.yaml @@ -0,0 +1,49 @@ +concurrency: 256 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 256 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 2560 + requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/result_summary.json new file mode 100644 index 000000000..8fdf6e29c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 3735858007321.0, + "ttft": { + "total": 7190722621440.0, + "percentiles": { + "50": 292591252.5, + "95": 370424039.25, + "90": 354857481.9 + } + }, + "output_sequence_lengths": { + "total": 7211649.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1930.3863, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/system_desc.json new file mode 100644 index 000000000..1ea55431b --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-D", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_8chip", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.657651 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/config.yaml new file mode 100644 index 000000000..b03f20973 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/config.yaml @@ -0,0 +1 @@ +concurrency: 32 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/point.yaml new file mode 100644 index 000000000..0a689acc1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/point.yaml @@ -0,0 +1,49 @@ +concurrency: 32 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 32 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 320 + requests_issued: 320 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/result_summary.json new file mode 100644 index 000000000..bfb87ca1f --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 12616634802170.0, + "ttft": { + "total": 3350546350080.0, + "percentiles": { + "50": 136334080.0, + "95": 193898324.0, + "90": 182385475.2 + } + }, + "output_sequence_lengths": { + "total": 7189678.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 569.857, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/system_desc.json new file mode 100644 index 000000000..c5c549d00 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-D", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_8chip", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.194141 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/config.yaml new file mode 100644 index 000000000..846832e5e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/config.yaml @@ -0,0 +1 @@ +concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/point.yaml new file mode 100644 index 000000000..779d0205d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/point.yaml @@ -0,0 +1,49 @@ +concurrency: 512 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 512 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 5120 + requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/result_summary.json new file mode 100644 index 000000000..4107afa33 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 2852962195525.0, + "ttft": { + "total": 10591377063936.0, + "percentiles": { + "50": 430964236.0, + "95": 646502413.0, + "90": 603394777.6 + } + }, + "output_sequence_lengths": { + "total": 7193097.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 2521.2732, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/system_desc.json new file mode 100644 index 000000000..bc660162a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-D", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_8chip", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.858956 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/config.yaml new file mode 100644 index 000000000..7db6216f5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/config.yaml @@ -0,0 +1 @@ +concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/point.yaml new file mode 100644 index 000000000..e67aff2c5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/point.yaml @@ -0,0 +1,49 @@ +concurrency: 64 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 64 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 640 + requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/result_summary.json new file mode 100644 index 000000000..7916313e6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 7605866967850.0, + "ttft": { + "total": 4213553332224.0, + "percentiles": { + "50": 171449924.0, + "95": 229212318.25, + "90": 217659839.4 + } + }, + "output_sequence_lengths": { + "total": 7198828.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 946.4836, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/system_desc.json new file mode 100644 index 000000000..5f171ac62 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-D", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_8chip", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.322451 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/accuracy_results.json new file mode 100644 index 000000000..9806cebd4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "humaneval": { + "dataset_name": "humaneval", + "num_samples": 500, + "score": 0.45, + "extractor": "ExactMatchExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/config.yaml new file mode 100644 index 000000000..e1b56a7e8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/config.yaml @@ -0,0 +1 @@ +concurrency: 8 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/point.yaml new file mode 100644 index 000000000..2ee5a330c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/point.yaml @@ -0,0 +1,49 @@ +concurrency: 8 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: qwen3-coder-480b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 8 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 80 + requests_issued: 80 diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/result_summary.json new file mode 100644 index 000000000..e82581895 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 40699327677953.0, + "ttft": { + "total": 3014703452160.0, + "percentiles": { + "50": 122668597.5, + "95": 178853483.5, + "90": 167616506.3 + } + }, + "output_sequence_lengths": { + "total": 7176538.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 176.3306, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/system_desc.json new file mode 100644 index 000000000..c33ccc8a7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-D", + "submitter_contact": "contact@example.com", + "system_name": "sys_tpu_8chip", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM (TPU)", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Custom TPU Host CPU", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "host_memory_capacity": "192 GB", + "host_networking": "Ethernet 100Gb", + "host_storage_capacity": "10 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Debian 11", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Google TPU v5p", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "qwen3-coder-480b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.060073 +} diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/system_power.json b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/system_power.json new file mode 100644 index 000000000..5f6438327 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/system_power.json @@ -0,0 +1,44 @@ +{ + "system_desc_id": "sys_tpu_8chip", + "cooling": "liquid", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": { + "model": "Custom TPU Host CPU", + "count_per_node": 1, + "tdp_per_unit": { + "value_w": 350, + "source_type": "vendor_spec", + "source": "https://example.com/cpu-spec" + } + }, + "accelerator": { + "model": "Google TPU v5p", + "count_per_node": 8, + "tdp_per_unit": { + "value_w": 700, + "source_type": "vendor_spec", + "source": "https://example.com/accelerator-spec" + } + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": { + "value_w": 3500, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + } + } + ], + "scale_out": { + "present": false + } +} diff --git a/tests/fixtures/validation/submissions/sub_d/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_d/src/vllm/README.md new file mode 100644 index 000000000..be3edfb30 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_d/src/vllm/README.md @@ -0,0 +1,3 @@ +# vllm + +Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_e/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_e/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/config.yaml new file mode 100644 index 000000000..6d8e711ba --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/config.yaml @@ -0,0 +1 @@ +concurrency: 1 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/point.yaml new file mode 100644 index 000000000..e54fdc8c2 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/point.yaml @@ -0,0 +1,49 @@ +concurrency: 1 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 10 + requests_issued: 10 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/result_summary.json new file mode 100644 index 000000000..30d450bb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 25761392228349.0, + "ttft": { + "total": 1364073025980.0, + "percentiles": { + "50": 102040172.5, + "95": 239159702.6, + "90": 211735796.57999998 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 66.4212, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/system_desc.json new file mode 100644 index 000000000..2345b5af3 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-E", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp1", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.019947 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/config.yaml new file mode 100644 index 000000000..b8581faa1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/config.yaml @@ -0,0 +1 @@ +concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/point.yaml new file mode 100644 index 000000000..c0b85f7fc --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/point.yaml @@ -0,0 +1,50 @@ +concurrency: 1024 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +offline: elected +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1024 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 10240 + requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/result_summary.json new file mode 100644 index 000000000..64ae3c688 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 515488552187.99994, + "ttft": { + "total": 108213125776644.0, + "percentiles": { + "50": 8094937595.5, + "95": 17493931139.45, + "90": 15614132430.66 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 3319.3792, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/system_desc.json new file mode 100644 index 000000000..364f967cb --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-E", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp1", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.99684 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/config.yaml new file mode 100644 index 000000000..51457c3cf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/config.yaml @@ -0,0 +1 @@ +concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/point.yaml new file mode 100644 index 000000000..02b3bf995 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/point.yaml @@ -0,0 +1,49 @@ +concurrency: 128 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 128 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 1280 + requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/result_summary.json new file mode 100644 index 000000000..29712806d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 627339011078.0, + "ttft": { + "total": 2788710324048.0, + "percentiles": { + "50": 208610886.0, + "95": 438744546.99999994, + "90": 392717814.79999995 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 2727.5555, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/system_desc.json new file mode 100644 index 000000000..146f52163 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-E", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp1", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.81911 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/config.yaml new file mode 100644 index 000000000..f9afe658a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/config.yaml @@ -0,0 +1 @@ +concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/point.yaml new file mode 100644 index 000000000..ccc4635a1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/point.yaml @@ -0,0 +1,49 @@ +concurrency: 16 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 16 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 160 + requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/result_summary.json new file mode 100644 index 000000000..f2614d79c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 2064726029363.0, + "ttft": { + "total": 1680393412872.0, + "percentiles": { + "50": 125702679.0, + "95": 274517621.75, + "90": 244754633.2 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 828.7308, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/system_desc.json new file mode 100644 index 000000000..d451053ae --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-E", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp1", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.248875 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/config.yaml new file mode 100644 index 000000000..ccc7a88ba --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/config.yaml @@ -0,0 +1 @@ +concurrency: 2 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/point.yaml new file mode 100644 index 000000000..3e2da1541 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/point.yaml @@ -0,0 +1,49 @@ +concurrency: 2 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 2 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 20 + requests_issued: 20 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/result_summary.json new file mode 100644 index 000000000..800a3470d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 13163162954524.0, + "ttft": { + "total": 1484351746344.0, + "percentiles": { + "50": 111037683.0, + "95": 243860930.6, + "90": 217296281.07999998 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 129.9917, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/system_desc.json new file mode 100644 index 000000000..477083217 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-E", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp1", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.039038 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/config.yaml new file mode 100644 index 000000000..f928653b5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/config.yaml @@ -0,0 +1 @@ +concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/point.yaml new file mode 100644 index 000000000..8e3d1b5e7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/point.yaml @@ -0,0 +1,49 @@ +concurrency: 256 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 256 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 2560 + requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/result_summary.json new file mode 100644 index 000000000..565ee9ee8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 543519373006.00006, + "ttft": { + "total": 3961749503400.0, + "percentiles": { + "50": 296360675.0, + "95": 614244970.3, + "90": 550668111.24 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 3148.1895, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/system_desc.json new file mode 100644 index 000000000..4f3a56426 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-E", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp1", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.94543 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/config.yaml new file mode 100644 index 000000000..b03f20973 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/config.yaml @@ -0,0 +1 @@ +concurrency: 32 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/point.yaml new file mode 100644 index 000000000..b2ff8785c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/point.yaml @@ -0,0 +1,49 @@ +concurrency: 32 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 32 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 320 + requests_issued: 320 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/result_summary.json new file mode 100644 index 000000000..d2d18057a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 1210809443938.0, + "ttft": { + "total": 1830336280008.0, + "percentiles": { + "50": 136919231.0, + "95": 299232559.5, + "90": 266769893.8 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1413.1885, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/system_desc.json new file mode 100644 index 000000000..6af50687a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-E", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp1", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.424393 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/config.yaml new file mode 100644 index 000000000..24d4dedbd --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/config.yaml @@ -0,0 +1 @@ +concurrency: 4 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/point.yaml new file mode 100644 index 000000000..532ef974c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/point.yaml @@ -0,0 +1,49 @@ +concurrency: 4 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 4 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 40 + requests_issued: 40 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/result_summary.json new file mode 100644 index 000000000..1e9c9a569 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 6914982314847.001, + "ttft": { + "total": 1477899507360.0, + "percentiles": { + "50": 110555020.0, + "95": 246967133.4, + "90": 219684710.72000003 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 247.4485, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/system_desc.json new file mode 100644 index 000000000..cd43cda38 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-E", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp1", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.074311 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/config.yaml new file mode 100644 index 000000000..846832e5e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/config.yaml @@ -0,0 +1 @@ +concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/point.yaml new file mode 100644 index 000000000..65e950b29 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/point.yaml @@ -0,0 +1,49 @@ +concurrency: 512 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 512 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 5120 + requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/result_summary.json new file mode 100644 index 000000000..f5048b17f --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 513859639398.0, + "ttft": { + "total": 6792119454924.0, + "percentiles": { + "50": 508087930.5, + "95": 2020109026.3999984, + "90": 1717704807.2199988 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 3329.9015, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/system_desc.json new file mode 100644 index 000000000..eccdac0a3 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-E", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp1", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 1.0 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/config.yaml new file mode 100644 index 000000000..7db6216f5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/config.yaml @@ -0,0 +1 @@ +concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/point.yaml new file mode 100644 index 000000000..148978304 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/point.yaml @@ -0,0 +1,49 @@ +concurrency: 64 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 64 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 640 + requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/result_summary.json new file mode 100644 index 000000000..06263731f --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 817907368671.0, + "ttft": { + "total": 2259057999168.0, + "percentiles": { + "50": 168989976.0, + "95": 370875724.7999998, + "90": 330498575.03999984 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 2092.0486, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/system_desc.json new file mode 100644 index 000000000..f236ea7d7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-E", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp1", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.628261 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/config.yaml new file mode 100644 index 000000000..e1b56a7e8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/config.yaml @@ -0,0 +1 @@ +concurrency: 8 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/point.yaml new file mode 100644 index 000000000..46a76d627 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/point.yaml @@ -0,0 +1,49 @@ +concurrency: 8 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 8 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 80 + requests_issued: 80 diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/result_summary.json new file mode 100644 index 000000000..2cbec8f84 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 3722968524345.0, + "ttft": { + "total": 1551415687080.0, + "percentiles": { + "50": 116054435.0, + "95": 253079730.75, + "90": 225674671.60000002 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 459.6069, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/system_desc.json new file mode 100644 index 000000000..69d3fab9a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-E", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp1", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.138024 +} diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/system_power.json b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/system_power.json new file mode 100644 index 000000000..3d133a620 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/system_power.json @@ -0,0 +1,44 @@ +{ + "system_desc_id": "sys_gaudi_dp1", + "cooling": "air", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": { + "model": "Intel Xeon Platinum 8480+", + "count_per_node": 2, + "tdp_per_unit": { + "value_w": 350, + "source_type": "vendor_spec", + "source": "https://example.com/cpu-spec" + } + }, + "accelerator": { + "model": "Intel Gaudi 3", + "count_per_node": 8, + "tdp_per_unit": { + "value_w": 700, + "source_type": "vendor_spec", + "source": "https://example.com/accelerator-spec" + } + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": { + "value_w": 3500, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + } + } + ], + "scale_out": { + "present": false + } +} diff --git a/tests/fixtures/validation/submissions/sub_e/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_e/src/vllm/README.md new file mode 100644 index 000000000..be3edfb30 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_e/src/vllm/README.md @@ -0,0 +1,3 @@ +# vllm + +Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_f/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_f/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/config.yaml new file mode 100644 index 000000000..6d8e711ba --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/config.yaml @@ -0,0 +1 @@ +concurrency: 1 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/point.yaml new file mode 100644 index 000000000..e54fdc8c2 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/point.yaml @@ -0,0 +1,49 @@ +concurrency: 1 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 10 + requests_issued: 10 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/result_summary.json new file mode 100644 index 000000000..4bb4b9092 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 45583460351935.0, + "ttft": { + "total": 2574627124512.0, + "percentiles": { + "50": 192596284.0, + "95": 449459125.5, + "90": 398086557.20000005 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 37.5378, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/system_desc.json new file mode 100644 index 000000000..25be44d22 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-F", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp2", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 2, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 2, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 2, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.022095 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/config.yaml new file mode 100644 index 000000000..b8581faa1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/config.yaml @@ -0,0 +1 @@ +concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/point.yaml new file mode 100644 index 000000000..c0b85f7fc --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/point.yaml @@ -0,0 +1,50 @@ +concurrency: 1024 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +offline: elected +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1024 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 10240 + requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/result_summary.json new file mode 100644 index 000000000..f96d2785c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 1011902488209.0, + "ttft": { + "total": 479444018825568.0, + "percentiles": { + "50": 35865052276.0, + "95": 46774478776.74999, + "90": 44592593476.59999 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1690.9752, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/system_desc.json new file mode 100644 index 000000000..7590b1554 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-F", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp2", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 2, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 2, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 2, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.995315 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/config.yaml new file mode 100644 index 000000000..51457c3cf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/config.yaml @@ -0,0 +1 @@ +concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/point.yaml new file mode 100644 index 000000000..02b3bf995 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/point.yaml @@ -0,0 +1,49 @@ +concurrency: 128 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 128 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 1280 + requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/result_summary.json new file mode 100644 index 000000000..9fd9647c8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 1192600590769.0, + "ttft": { + "total": 5379052027176.0, + "percentiles": { + "50": 402382707.0, + "95": 869390356.9, + "90": 775988826.9200001 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1434.7653, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/system_desc.json new file mode 100644 index 000000000..f534f1db0 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-F", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp2", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 2, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 2, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 2, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.844509 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/config.yaml new file mode 100644 index 000000000..f9afe658a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/config.yaml @@ -0,0 +1 @@ +concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/point.yaml new file mode 100644 index 000000000..ccc4635a1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/point.yaml @@ -0,0 +1,49 @@ +concurrency: 16 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 16 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 160 + requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/result_summary.json new file mode 100644 index 000000000..9c907163d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 3505224837302.0005, + "ttft": { + "total": 3373550899608.0, + "percentiles": { + "50": 252360181.0, + "95": 550608223.3999999, + "90": 490958614.9199999 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 488.1576, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/system_desc.json new file mode 100644 index 000000000..698421adf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-F", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp2", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 2, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 2, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 2, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.287332 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/config.yaml new file mode 100644 index 000000000..ccc7a88ba --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/config.yaml @@ -0,0 +1 @@ +concurrency: 2 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/point.yaml new file mode 100644 index 000000000..3e2da1541 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/point.yaml @@ -0,0 +1,49 @@ +concurrency: 2 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 2 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 20 + requests_issued: 20 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/result_summary.json new file mode 100644 index 000000000..29b907f0d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 23023578002485.996, + "ttft": { + "total": 2892442942776.0, + "percentiles": { + "50": 216370657.0, + "95": 457229317.34999996, + "90": 409057585.28 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 74.3196, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/system_desc.json new file mode 100644 index 000000000..af2bdc05e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-F", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp2", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 2, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 2, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 2, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.043745 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/config.yaml new file mode 100644 index 000000000..f928653b5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/config.yaml @@ -0,0 +1 @@ +concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/point.yaml new file mode 100644 index 000000000..8e3d1b5e7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/point.yaml @@ -0,0 +1,49 @@ +concurrency: 256 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 256 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 2560 + requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/result_summary.json new file mode 100644 index 000000000..69e9f5c92 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 1045116285522.0, + "ttft": { + "total": 7725994840968.0, + "percentiles": { + "50": 577946951.0, + "95": 1242402278.2499998, + "90": 1109511212.7999997 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1637.236, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/system_desc.json new file mode 100644 index 000000000..a52613032 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-F", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp2", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 2, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 2, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 2, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.963684 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/config.yaml new file mode 100644 index 000000000..b03f20973 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/config.yaml @@ -0,0 +1 @@ +concurrency: 32 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/point.yaml new file mode 100644 index 000000000..b2ff8785c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/point.yaml @@ -0,0 +1,49 @@ +concurrency: 32 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 32 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 320 + requests_issued: 320 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/result_summary.json new file mode 100644 index 000000000..744dda378 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 2182110754084.9998, + "ttft": { + "total": 3596106784332.0, + "percentiles": { + "50": 269008586.5, + "95": 604284570.0, + "90": 537229373.3 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 784.1499, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/system_desc.json new file mode 100644 index 000000000..7299e888c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-F", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp2", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 2, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 2, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 2, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.461554 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/config.yaml new file mode 100644 index 000000000..24d4dedbd --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/config.yaml @@ -0,0 +1 @@ +concurrency: 4 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/point.yaml new file mode 100644 index 000000000..532ef974c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/point.yaml @@ -0,0 +1,49 @@ +concurrency: 4 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 4 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 40 + requests_issued: 40 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/result_summary.json new file mode 100644 index 000000000..8e9864bd0 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 11622328296428.0, + "ttft": { + "total": 2922014582988.0, + "percentiles": { + "50": 218582778.5, + "95": 470498090.7499999, + "90": 420115028.29999995 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 147.2254, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/system_desc.json new file mode 100644 index 000000000..a53c325dd --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-F", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp2", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 2, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 2, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 2, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.086657 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/config.yaml new file mode 100644 index 000000000..846832e5e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/config.yaml @@ -0,0 +1 @@ +concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/point.yaml new file mode 100644 index 000000000..65e950b29 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/point.yaml @@ -0,0 +1,49 @@ +concurrency: 512 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 512 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 5120 + requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/result_summary.json new file mode 100644 index 000000000..a2c730ba8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 1007161846898.0, + "ttft": { + "total": 19342596948528.0, + "percentiles": { + "50": 1446932746.0, + "95": 17416985904.85, + "90": 14222975273.08 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1698.9345, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/system_desc.json new file mode 100644 index 000000000..14a208ef5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-F", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp2", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 2, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 2, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 2, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 1.0 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/config.yaml new file mode 100644 index 000000000..7db6216f5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/config.yaml @@ -0,0 +1 @@ +concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/point.yaml new file mode 100644 index 000000000..148978304 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/point.yaml @@ -0,0 +1,49 @@ +concurrency: 64 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 64 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 640 + requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/result_summary.json new file mode 100644 index 000000000..810894306 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 1528920443390.0, + "ttft": { + "total": 4340911548228.0, + "percentiles": { + "50": 324724083.5, + "95": 700762596.05, + "90": 625554893.54 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1119.157, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/system_desc.json new file mode 100644 index 000000000..2971eb8aa --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-F", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp2", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 2, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 2, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 2, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.658741 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/accuracy_results.json new file mode 100644 index 000000000..cd085ffb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/accuracy_results.json @@ -0,0 +1,15 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 13368, + "score": { + "rouge1": 38.7792, + "rouge2": 15.9075, + "rougel": 24.4957, + "rougelsum": 35.793, + "gen_len": 8167644.0 + }, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/config.yaml new file mode 100644 index 000000000..e1b56a7e8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/config.yaml @@ -0,0 +1 @@ +concurrency: 8 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/point.yaml new file mode 100644 index 000000000..46a76d627 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/point.yaml @@ -0,0 +1,49 @@ +concurrency: 8 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1024 +model_name: llama3-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 8 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 80 + requests_issued: 80 diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/result_summary.json new file mode 100644 index 000000000..667db8712 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 13368, + "n_samples_completed": 13368, + "n_samples_failed": 0, + "duration_ns": 6211960687374.0, + "ttft": { + "total": 3152037458208.0, + "percentiles": { + "50": 235789756.0, + "95": 509306917.05, + "90": 454603484.84000003 + } + }, + "output_sequence_lengths": { + "total": 1711102.0, + "percentiles": {} + }, + "tpot": { + "total": 54127032000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 275.4528, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/system_desc.json new file mode 100644 index 000000000..3a351b933 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/system_desc.json @@ -0,0 +1,60 @@ +{ + "submitter_org_names": "Submitter-F", + "submitter_contact": "contact@example.com", + "system_name": "sys_gaudi_dp2", + "system_category": "datacenter", + "max_supported_concurrency": 1024, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 2, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "512 GB", + "host_networking": "Ethernet 200Gb", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "Intel Gaudi 3", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "96 GB HBM2e", + "accelerator_memory_type": "HBM3e" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 2, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 2, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.162133 +} diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/system_power.json b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/system_power.json new file mode 100644 index 000000000..7306e7adb --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/system_power.json @@ -0,0 +1,68 @@ +{ + "system_desc_id": "sys_gaudi_dp2", + "cooling": "air", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": 2, + "power_method": "component_sum", + "components": { + "cpu": { + "model": "Intel Xeon Platinum 8480+", + "count_per_node": 2, + "tdp_per_unit": { + "value_w": 350, + "source_type": "vendor_spec", + "source": "https://example.com/cpu-spec" + } + }, + "accelerator": { + "model": "Intel Gaudi 3", + "count_per_node": 8, + "tdp_per_unit": { + "value_w": 700, + "source_type": "vendor_spec", + "source": "https://example.com/accelerator-spec" + } + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": { + "value_w": 3500, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + } + } + ], + "scale_out": { + "present": true, + "cabling": "passive", + "required_bandwidth_tbps": 6.4, + "nics": { + "count_per_node": 8, + "bandwidth_per_nic_gbps": 400, + "counted": true, + "tdp_per_nic": { + "value_w": 25, + "source_type": "vendor_spec", + "source": "https://example.com/nic-spec" + } + }, + "switches": [ + { + "model": "NVIDIA Spectrum SN5610", + "count": 1, + "bandwidth_tbps": 51.2, + "power_per_switch": { + "value_w": 900, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + ] + } +} diff --git a/tests/fixtures/validation/submissions/sub_f/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_f/src/vllm/README.md new file mode 100644 index 000000000..be3edfb30 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_f/src/vllm/README.md @@ -0,0 +1,3 @@ +# vllm + +Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_g/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_g/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/config.yaml new file mode 100644 index 000000000..b8581faa1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/config.yaml @@ -0,0 +1 @@ +concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/point.yaml new file mode 100644 index 000000000..402f06b6f --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/point.yaml @@ -0,0 +1,49 @@ +concurrency: 1024 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1024 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 10240 + requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/result_summary.json new file mode 100644 index 000000000..0e7573814 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 358433737274.0, + "ttft": { + "total": 5094461190144.0, + "percentiles": { + "50": 207294156.5, + "95": 475022456.0, + "90": 421476796.1 + } + }, + "output_sequence_lengths": { + "total": 5972600.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 16663.052, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/system_desc.json new file mode 100644 index 000000000..b4218ddb6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-G", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_vllm", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.979024 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/config.yaml new file mode 100644 index 000000000..51457c3cf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/config.yaml @@ -0,0 +1 @@ +concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/point.yaml new file mode 100644 index 000000000..d3e80b258 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/point.yaml @@ -0,0 +1,49 @@ +concurrency: 128 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 128 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 1280 + requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/result_summary.json new file mode 100644 index 000000000..7bb8379f6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 1154776483445.0, + "ttft": { + "total": 2436995543040.0, + "percentiles": { + "50": 99161602.5, + "95": 649174965.25, + "90": 539172292.7 + } + }, + "output_sequence_lengths": { + "total": 5954161.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 5156.1156, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/system_desc.json new file mode 100644 index 000000000..2bc9f37c3 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-G", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_vllm", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.302943 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/config.yaml new file mode 100644 index 000000000..087256dee --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/config.yaml @@ -0,0 +1 @@ +concurrency: 1536 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/point.yaml new file mode 100644 index 000000000..384d2fa43 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/point.yaml @@ -0,0 +1,49 @@ +concurrency: 1536 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1536 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 15360 + requests_issued: 15360 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/result_summary.json new file mode 100644 index 000000000..c57070abe --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 371427380557.0, + "ttft": { + "total": 6655861899264.0, + "percentiles": { + "50": 270827714.0, + "95": 2812902801.5, + "90": 2304487784.0 + } + }, + "output_sequence_lengths": { + "total": 5972600.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 16080.1285, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/system_desc.json new file mode 100644 index 000000000..44567a339 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-G", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_vllm", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.944775 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/config.yaml new file mode 100644 index 000000000..5ace5b44d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/config.yaml @@ -0,0 +1 @@ +concurrency: 192 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/point.yaml new file mode 100644 index 000000000..d0968cfb8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/point.yaml @@ -0,0 +1,49 @@ +concurrency: 192 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 192 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 1920 + requests_issued: 1920 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/result_summary.json new file mode 100644 index 000000000..1a0978c61 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 1008814134637.0, + "ttft": { + "total": 2930111361024.0, + "percentiles": { + "50": 119226536.5, + "95": 610194947.75, + "90": 512001265.5 + } + }, + "output_sequence_lengths": { + "total": 5970998.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 5918.8286, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/system_desc.json new file mode 100644 index 000000000..2d1302929 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-G", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_vllm", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.347756 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/config.yaml new file mode 100644 index 000000000..6b53d1bba --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/config.yaml @@ -0,0 +1 @@ +concurrency: 2048 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/point.yaml new file mode 100644 index 000000000..a1ed6ee7e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/point.yaml @@ -0,0 +1,50 @@ +concurrency: 2048 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +offline: elected +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 2048 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 20480 + requests_issued: 20480 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/result_summary.json new file mode 100644 index 000000000..bb4623007 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 350915227928.0, + "ttft": { + "total": 7688238305280.0, + "percentiles": { + "50": 312835217.5, + "95": 4378115873.25, + "90": 3565059742.1000004 + } + }, + "output_sequence_lengths": { + "total": 5972600.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 17020.065, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/system_desc.json new file mode 100644 index 000000000..90e2891d6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-G", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_vllm", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 1.0 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/config.yaml new file mode 100644 index 000000000..f928653b5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/config.yaml @@ -0,0 +1 @@ +concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/point.yaml new file mode 100644 index 000000000..d7a6a9b5c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/point.yaml @@ -0,0 +1,49 @@ +concurrency: 256 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 256 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 2560 + requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/result_summary.json new file mode 100644 index 000000000..36daea8a5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 776418597591.0, + "ttft": { + "total": 3292125204480.0, + "percentiles": { + "50": 133956917.5, + "95": 590184585.25, + "90": 498939051.70000005 + } + }, + "output_sequence_lengths": { + "total": 5962724.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 7679.78, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/system_desc.json new file mode 100644 index 000000000..3881de41f --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-G", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_vllm", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.451219 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/config.yaml new file mode 100644 index 000000000..05556e355 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/config.yaml @@ -0,0 +1 @@ +concurrency: 384 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/point.yaml new file mode 100644 index 000000000..22cad9262 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/point.yaml @@ -0,0 +1,49 @@ +concurrency: 384 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 384 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 3840 + requests_issued: 3840 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/result_summary.json new file mode 100644 index 000000000..ea56360b8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 517816400500.0, + "ttft": { + "total": 3370839257088.0, + "percentiles": { + "50": 137159800.5, + "95": 434521706.0, + "90": 375049324.9 + } + }, + "output_sequence_lengths": { + "total": 5964063.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 11517.7175, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/system_desc.json new file mode 100644 index 000000000..5f75c0c76 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-G", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_vllm", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.676714 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/config.yaml new file mode 100644 index 000000000..846832e5e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/config.yaml @@ -0,0 +1 @@ +concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/point.yaml new file mode 100644 index 000000000..d8a2668be --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/point.yaml @@ -0,0 +1,49 @@ +concurrency: 512 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 512 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 5120 + requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/result_summary.json new file mode 100644 index 000000000..ee10d7260 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 444594042812.0, + "ttft": { + "total": 3720757702656.0, + "percentiles": { + "50": 151398018.5, + "95": 409875781.5, + "90": 358180228.9 + } + }, + "output_sequence_lengths": { + "total": 5961100.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 13407.9619, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/system_desc.json new file mode 100644 index 000000000..3429cd1e1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-G", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_vllm", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.787774 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/config.yaml new file mode 100644 index 000000000..7db6216f5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/config.yaml @@ -0,0 +1 @@ +concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/point.yaml new file mode 100644 index 000000000..b7140d590 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/point.yaml @@ -0,0 +1,49 @@ +concurrency: 64 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 64 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 640 + requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/result_summary.json new file mode 100644 index 000000000..878b5a8d4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 2145983999331.0002, + "ttft": { + "total": 2184279601152.0, + "percentiles": { + "50": 88878564.5, + "95": 958562205.25, + "90": 784625477.1 + } + }, + "output_sequence_lengths": { + "total": 5979513.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 2786.3735, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/system_desc.json new file mode 100644 index 000000000..dae732709 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-G", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_vllm", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.163711 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/config.yaml new file mode 100644 index 000000000..240492592 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/config.yaml @@ -0,0 +1 @@ +concurrency: 768 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/point.yaml new file mode 100644 index 000000000..8949daa55 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/point.yaml @@ -0,0 +1,49 @@ +concurrency: 768 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 768 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 7680 + requests_issued: 7680 diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/result_summary.json new file mode 100644 index 000000000..24c446ca5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 388271410974.0, + "ttft": { + "total": 4313087397888.0, + "percentiles": { + "50": 175499975.5, + "95": 426849699.25, + "90": 376579754.5 + } + }, + "output_sequence_lengths": { + "total": 5972600.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 15382.5387, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/system_desc.json new file mode 100644 index 000000000..bb5740b64 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-G", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_vllm", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "vLLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.903788 +} diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/system_power.json b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/system_power.json new file mode 100644 index 000000000..4d161c256 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/system_power.json @@ -0,0 +1,44 @@ +{ + "system_desc_id": "sys_8gpu_vllm", + "cooling": "air", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": { + "model": "Intel Xeon Platinum 8480+", + "count_per_node": 2, + "tdp_per_unit": { + "value_w": 350, + "source_type": "vendor_spec", + "source": "https://example.com/cpu-spec" + } + }, + "accelerator": { + "model": "NVIDIA H100-SXM5-80GB", + "count_per_node": 8, + "tdp_per_unit": { + "value_w": 700, + "source_type": "vendor_spec", + "source": "https://example.com/accelerator-spec" + } + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": { + "value_w": 3500, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + } + } + ], + "scale_out": { + "present": false + } +} diff --git a/tests/fixtures/validation/submissions/sub_g/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_g/src/vllm/README.md new file mode 100644 index 000000000..be3edfb30 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_g/src/vllm/README.md @@ -0,0 +1,3 @@ +# vllm + +Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_h/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_h/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/config.yaml new file mode 100644 index 000000000..b8581faa1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/config.yaml @@ -0,0 +1 @@ +concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/point.yaml new file mode 100644 index 000000000..402f06b6f --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/point.yaml @@ -0,0 +1,49 @@ +concurrency: 1024 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1024 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 10240 + requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/result_summary.json new file mode 100644 index 000000000..fcbea3f4d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 392249945920.0, + "ttft": { + "total": 3834814304256.0, + "percentiles": { + "50": 156038993.5, + "95": 510605960.25, + "90": 439692566.90000004 + } + }, + "output_sequence_lengths": { + "total": 5958417.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 15190.3577, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/system_desc.json new file mode 100644 index 000000000..a21b45eef --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-H", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_sglang", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "SGLang", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 1.0 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/config.yaml new file mode 100644 index 000000000..51457c3cf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/config.yaml @@ -0,0 +1 @@ +concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/point.yaml new file mode 100644 index 000000000..d3e80b258 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/point.yaml @@ -0,0 +1,49 @@ +concurrency: 128 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 128 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 1280 + requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/result_summary.json new file mode 100644 index 000000000..842895879 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 1403030676854.0, + "ttft": { + "total": 2718343397376.0, + "percentiles": { + "50": 110609676.0, + "95": 758335350.75, + "90": 628790215.8 + } + }, + "output_sequence_lengths": { + "total": 5949986.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 4240.8096, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/system_desc.json new file mode 100644 index 000000000..68ad390a2 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-H", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_sglang", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "SGLang", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.279178 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/config.yaml new file mode 100644 index 000000000..087256dee --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/config.yaml @@ -0,0 +1 @@ +concurrency: 1536 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/point.yaml new file mode 100644 index 000000000..384d2fa43 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/point.yaml @@ -0,0 +1,49 @@ +concurrency: 1536 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1536 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 15360 + requests_issued: 15360 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/result_summary.json new file mode 100644 index 000000000..6de815aba --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 402038739154.0, + "ttft": { + "total": 4694270042112.0, + "percentiles": { + "50": 191010337.0, + "95": 2130321446.0, + "90": 1742459224.2 + } + }, + "output_sequence_lengths": { + "total": 5958417.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 14820.5046, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/system_desc.json new file mode 100644 index 000000000..881ef25e5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-H", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_sglang", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "SGLang", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.975652 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/config.yaml new file mode 100644 index 000000000..5ace5b44d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/config.yaml @@ -0,0 +1 @@ +concurrency: 192 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/point.yaml new file mode 100644 index 000000000..d0968cfb8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/point.yaml @@ -0,0 +1,49 @@ +concurrency: 192 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 192 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 1920 + requests_issued: 1920 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/result_summary.json new file mode 100644 index 000000000..9410fd05f --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 1001832049974.0001, + "ttft": { + "total": 2768901292032.0, + "percentiles": { + "50": 112666882.0, + "95": 635672662.0, + "90": 531071506.0 + } + }, + "output_sequence_lengths": { + "total": 5956127.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 5945.235, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/system_desc.json new file mode 100644 index 000000000..fd90f338e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-H", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_sglang", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "SGLang", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.391382 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/config.yaml new file mode 100644 index 000000000..6b53d1bba --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/config.yaml @@ -0,0 +1 @@ +concurrency: 2048 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/point.yaml new file mode 100644 index 000000000..a1ed6ee7e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/point.yaml @@ -0,0 +1,50 @@ +concurrency: 2048 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +offline: elected +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 2048 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 20480 + requests_issued: 20480 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/result_summary.json new file mode 100644 index 000000000..abd1561a7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 409541274676.0, + "ttft": { + "total": 5435139072000.0, + "percentiles": { + "50": 221156375.0, + "95": 4408367949.0, + "90": 3570925634.2000003 + } + }, + "output_sequence_lengths": { + "total": 5958417.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 14549.0024, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/system_desc.json new file mode 100644 index 000000000..99d06ae1a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-H", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_sglang", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "SGLang", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.957779 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/config.yaml new file mode 100644 index 000000000..f928653b5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/config.yaml @@ -0,0 +1 @@ +concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/point.yaml new file mode 100644 index 000000000..d7a6a9b5c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/point.yaml @@ -0,0 +1,49 @@ +concurrency: 256 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 256 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 2560 + requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/result_summary.json new file mode 100644 index 000000000..7129f0901 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 799717038355.0, + "ttft": { + "total": 2881879240704.0, + "percentiles": { + "50": 117263966.5, + "95": 558828541.0, + "90": 470515626.1 + } + }, + "output_sequence_lengths": { + "total": 5960490.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 7453.2487, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/system_desc.json new file mode 100644 index 000000000..7ab582744 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-H", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_sglang", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "SGLang", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.490657 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/config.yaml new file mode 100644 index 000000000..05556e355 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/config.yaml @@ -0,0 +1 @@ +concurrency: 384 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/point.yaml new file mode 100644 index 000000000..22cad9262 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/point.yaml @@ -0,0 +1,49 @@ +concurrency: 384 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 384 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 3840 + requests_issued: 3840 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/result_summary.json new file mode 100644 index 000000000..2cdfd5d7c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 600137480188.0, + "ttft": { + "total": 3042139607040.0, + "percentiles": { + "50": 123784977.5, + "95": 483974886.0, + "90": 411936904.3 + } + }, + "output_sequence_lengths": { + "total": 5958391.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 9928.3767, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/system_desc.json new file mode 100644 index 000000000..510e3b803 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-H", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_sglang", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "SGLang", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.653597 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/config.yaml new file mode 100644 index 000000000..846832e5e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/config.yaml @@ -0,0 +1 @@ +concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/point.yaml new file mode 100644 index 000000000..d8a2668be --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/point.yaml @@ -0,0 +1,49 @@ +concurrency: 512 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 512 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 5120 + requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/result_summary.json new file mode 100644 index 000000000..5bbd690dd --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 500684255434.00006, + "ttft": { + "total": 3142710755328.0, + "percentiles": { + "50": 127877228.0, + "95": 443205541.0, + "90": 380139878.4 + } + }, + "output_sequence_lengths": { + "total": 5958417.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 11900.548, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/system_desc.json new file mode 100644 index 000000000..770d456fc --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-H", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_sglang", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "SGLang", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.783428 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/config.yaml new file mode 100644 index 000000000..7db6216f5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/config.yaml @@ -0,0 +1 @@ +concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/point.yaml new file mode 100644 index 000000000..b7140d590 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/point.yaml @@ -0,0 +1,49 @@ +concurrency: 64 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 64 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 640 + requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/result_summary.json new file mode 100644 index 000000000..1ecf29e95 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 2608958278520.0, + "ttft": { + "total": 2562863296512.0, + "percentiles": { + "50": 104283174.5, + "95": 1100174349.75, + "90": 900996114.7 + } + }, + "output_sequence_lengths": { + "total": 5965062.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 2286.3769, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/system_desc.json new file mode 100644 index 000000000..b577f4c01 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-H", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_sglang", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "SGLang", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.150515 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/accuracy_results.json new file mode 100644 index 000000000..7242aa745 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/accuracy_results.json @@ -0,0 +1,9 @@ +{ + "cnn_dailymail_validation": { + "dataset_name": "cnn_dailymail_validation", + "num_samples": 500, + "score": 0.45, + "extractor": "RougeExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/config.yaml new file mode 100644 index 000000000..240492592 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/config.yaml @@ -0,0 +1 @@ +concurrency: 768 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/point.yaml new file mode 100644 index 000000000..8949daa55 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/point.yaml @@ -0,0 +1,49 @@ +concurrency: 768 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 2048 +model_name: llama3-70b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 768 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 7680 + requests_issued: 7680 diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/result_summary.json new file mode 100644 index 000000000..643f6985b --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 24576, + "n_samples_completed": 24576, + "n_samples_failed": 0, + "duration_ns": 513149916081.0, + "ttft": { + "total": 3919180652544.0, + "percentiles": { + "50": 159471869.0, + "95": 529036573.0, + "90": 455123632.2 + } + }, + "output_sequence_lengths": { + "total": 5958417.0, + "percentiles": {} + }, + "tpot": { + "total": 99508224000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 11611.4547, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/system_desc.json new file mode 100644 index 000000000..1b077fff0 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-H", + "submitter_contact": "contact@example.com", + "system_name": "sys_8gpu_sglang", + "system_category": "datacenter", + "max_supported_concurrency": 2048, + "serving_framework": "SGLang", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8480+", + "host_processors_per_node": 2, + "host_processor_core_count": 60, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100-SXM5-80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3-70b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.764396 +} diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/system_power.json b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/system_power.json new file mode 100644 index 000000000..c1490b516 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/system_power.json @@ -0,0 +1,44 @@ +{ + "system_desc_id": "sys_8gpu_sglang", + "cooling": "air", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": { + "model": "Intel Xeon Platinum 8480+", + "count_per_node": 2, + "tdp_per_unit": { + "value_w": 350, + "source_type": "vendor_spec", + "source": "https://example.com/cpu-spec" + } + }, + "accelerator": { + "model": "NVIDIA H100-SXM5-80GB", + "count_per_node": 8, + "tdp_per_unit": { + "value_w": 700, + "source_type": "vendor_spec", + "source": "https://example.com/accelerator-spec" + } + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": { + "value_w": 3500, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + } + } + ], + "scale_out": { + "present": false + } +} diff --git a/tests/fixtures/validation/submissions/sub_h/src/sglang/README.md b/tests/fixtures/validation/submissions/sub_h/src/sglang/README.md new file mode 100644 index 000000000..306956568 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_h/src/sglang/README.md @@ -0,0 +1,3 @@ +# sglang + +Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_i/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_i/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/config.yaml new file mode 100644 index 000000000..6d8e711ba --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/config.yaml @@ -0,0 +1 @@ +concurrency: 1 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/point.yaml new file mode 100644 index 000000000..0033075a4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/point.yaml @@ -0,0 +1,49 @@ +concurrency: 1 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 10 + requests_issued: 10 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/result_summary.json new file mode 100644 index 000000000..57e255517 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 29, + "n_samples_completed": 29, + "n_samples_failed": 0, + "duration_ns": 1810326781531.0, + "ttft": { + "total": 3239905752.0, + "percentiles": { + "50": 111720888.0, + "95": 121428535.79999998, + "90": 119487006.23999998 + } + }, + "output_sequence_lengths": { + "total": 168384.0, + "percentiles": {} + }, + "tpot": { + "total": 117421000.00000001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 93.013, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/system_desc.json new file mode 100644 index 000000000..d534f9e1d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/system_desc.json @@ -0,0 +1,62 @@ +{ + "submitter_org_names": "Submitter-I", + "submitter_contact": "contact@example.com", + "system_name": "sys_h200_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8562Y+", + "host_processors_per_node": 2, + "host_processor_core_count": 32, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H200-SXM-141GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "141 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.01422 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/config.yaml new file mode 100644 index 000000000..51457c3cf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/config.yaml @@ -0,0 +1 @@ +concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/point.yaml new file mode 100644 index 000000000..07b71d3b6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/point.yaml @@ -0,0 +1,49 @@ +concurrency: 128 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 128 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 1280 + requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/result_summary.json new file mode 100644 index 000000000..62ee53f3c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 1675, + "n_samples_completed": 1675, + "n_samples_failed": 0, + "duration_ns": 2363311856036.0, + "ttft": { + "total": 268304677475.0, + "percentiles": { + "50": 160181897.0, + "95": 456279775.1, + "90": 397060199.48 + } + }, + "output_sequence_lengths": { + "total": 6215384.0, + "percentiles": {} + }, + "tpot": { + "total": 6782075000.000001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 2629.9466, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/system_desc.json new file mode 100644 index 000000000..973226972 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/system_desc.json @@ -0,0 +1,62 @@ +{ + "submitter_org_names": "Submitter-I", + "submitter_contact": "contact@example.com", + "system_name": "sys_h200_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8562Y+", + "host_processors_per_node": 2, + "host_processor_core_count": 32, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H200-SXM-141GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "141 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.402063 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/config.yaml new file mode 100644 index 000000000..f9afe658a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/config.yaml @@ -0,0 +1 @@ +concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/point.yaml new file mode 100644 index 000000000..d1d3b5d43 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/point.yaml @@ -0,0 +1,49 @@ +concurrency: 16 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 16 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 160 + requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/result_summary.json new file mode 100644 index 000000000..d857471f2 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 339, + "n_samples_completed": 339, + "n_samples_failed": 0, + "duration_ns": 2003156043150.0, + "ttft": { + "total": 45912856545.0, + "percentiles": { + "50": 135436155.0, + "95": 241107175.0999999, + "90": 219972971.07999992 + } + }, + "output_sequence_lengths": { + "total": 1378214.0, + "percentiles": {} + }, + "tpot": { + "total": 1372611000.0000002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 688.0213, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/system_desc.json new file mode 100644 index 000000000..f785137f6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/system_desc.json @@ -0,0 +1,62 @@ +{ + "submitter_org_names": "Submitter-I", + "submitter_contact": "contact@example.com", + "system_name": "sys_h200_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8562Y+", + "host_processors_per_node": 2, + "host_processor_core_count": 32, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H200-SXM-141GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "141 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.105184 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/config.yaml new file mode 100644 index 000000000..ccc7a88ba --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/config.yaml @@ -0,0 +1 @@ +concurrency: 2 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/point.yaml new file mode 100644 index 000000000..a181e0641 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/point.yaml @@ -0,0 +1,49 @@ +concurrency: 2 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 2 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 20 + requests_issued: 20 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/result_summary.json new file mode 100644 index 000000000..0d10e7170 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 68, + "n_samples_completed": 68, + "n_samples_failed": 0, + "duration_ns": 1944103059102.0002, + "ttft": { + "total": 8684697316.0, + "percentiles": { + "50": 127716137.0, + "95": 137065360.35, + "90": 135195515.68 + } + }, + "output_sequence_lengths": { + "total": 315398.0, + "percentiles": {} + }, + "tpot": { + "total": 275332000.00000006, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 162.2332, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/system_desc.json new file mode 100644 index 000000000..aea985de0 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/system_desc.json @@ -0,0 +1,62 @@ +{ + "submitter_org_names": "Submitter-I", + "submitter_contact": "contact@example.com", + "system_name": "sys_h200_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8562Y+", + "host_processors_per_node": 2, + "host_processor_core_count": 32, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H200-SXM-141GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "141 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.024802 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/config.yaml new file mode 100644 index 000000000..f928653b5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/config.yaml @@ -0,0 +1 @@ +concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/point.yaml new file mode 100644 index 000000000..51eff9077 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/point.yaml @@ -0,0 +1,49 @@ +concurrency: 256 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 256 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 2560 + requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/result_summary.json new file mode 100644 index 000000000..381826878 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 2759, + "n_samples_completed": 2759, + "n_samples_failed": 0, + "duration_ns": 2418594571195.9995, + "ttft": { + "total": 512378878272.0, + "percentiles": { + "50": 185711808.0, + "95": 842228387.0, + "90": 710925071.2 + } + }, + "output_sequence_lengths": { + "total": 10495599.0, + "percentiles": {} + }, + "tpot": { + "total": 11171191000.000002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 4339.5446, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/system_desc.json new file mode 100644 index 000000000..3c0bc3e26 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/system_desc.json @@ -0,0 +1,62 @@ +{ + "submitter_org_names": "Submitter-I", + "submitter_contact": "contact@example.com", + "system_name": "sys_h200_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8562Y+", + "host_processors_per_node": 2, + "host_processor_core_count": 32, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H200-SXM-141GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "141 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.663424 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/config.yaml new file mode 100644 index 000000000..b03f20973 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/config.yaml @@ -0,0 +1 @@ +concurrency: 32 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/point.yaml new file mode 100644 index 000000000..8d7fd6807 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/point.yaml @@ -0,0 +1,49 @@ +concurrency: 32 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 32 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 320 + requests_issued: 320 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/result_summary.json new file mode 100644 index 000000000..000263e0d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 516, + "n_samples_completed": 516, + "n_samples_failed": 0, + "duration_ns": 2102160301502.9998, + "ttft": { + "total": 72592828674.0, + "percentiles": { + "50": 140683776.5, + "95": 237608990.0, + "90": 218223947.3 + } + }, + "output_sequence_lengths": { + "total": 1986475.0, + "percentiles": {} + }, + "tpot": { + "total": 2089284000.0000002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 944.9684, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/system_desc.json new file mode 100644 index 000000000..0e0a25fb4 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/system_desc.json @@ -0,0 +1,62 @@ +{ + "submitter_org_names": "Submitter-I", + "submitter_contact": "contact@example.com", + "system_name": "sys_h200_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8562Y+", + "host_processors_per_node": 2, + "host_processor_core_count": 32, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H200-SXM-141GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "141 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.144466 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/config.yaml new file mode 100644 index 000000000..24d4dedbd --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/config.yaml @@ -0,0 +1 @@ +concurrency: 4 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/point.yaml new file mode 100644 index 000000000..63c0feabc --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/point.yaml @@ -0,0 +1,49 @@ +concurrency: 4 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 4 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 40 + requests_issued: 40 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/result_summary.json new file mode 100644 index 000000000..ee8574091 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 117, + "n_samples_completed": 117, + "n_samples_failed": 0, + "duration_ns": 1962849595746.0, + "ttft": { + "total": 14924097162.0, + "percentiles": { + "50": 127556386.0, + "95": 135825117.4, + "90": 134171371.12 + } + }, + "output_sequence_lengths": { + "total": 539431.0, + "percentiles": {} + }, + "tpot": { + "total": 473733000.00000006, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 274.8203, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/system_desc.json new file mode 100644 index 000000000..a27ef4ed1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/system_desc.json @@ -0,0 +1,62 @@ +{ + "submitter_org_names": "Submitter-I", + "submitter_contact": "contact@example.com", + "system_name": "sys_h200_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8562Y+", + "host_processors_per_node": 2, + "host_processor_core_count": 32, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H200-SXM-141GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "141 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.042014 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/config.yaml new file mode 100644 index 000000000..846832e5e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/config.yaml @@ -0,0 +1 @@ +concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/point.yaml new file mode 100644 index 000000000..ac8f266dd --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/point.yaml @@ -0,0 +1,50 @@ +concurrency: 512 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +offline: elected +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 512 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 5120 + requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/result_summary.json new file mode 100644 index 000000000..e5a706061 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 4388, + "n_samples_completed": 4388, + "n_samples_failed": 0, + "duration_ns": 2594198991881.0, + "ttft": { + "total": 1184049108896.0, + "percentiles": { + "50": 269837992.0, + "95": 2367829187.25, + "90": 1948230948.2 + } + }, + "output_sequence_lengths": { + "total": 16968991.0, + "percentiles": {} + }, + "tpot": { + "total": 17767012000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 6541.1293, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/system_desc.json new file mode 100644 index 000000000..d06b9647b --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/system_desc.json @@ -0,0 +1,62 @@ +{ + "submitter_org_names": "Submitter-I", + "submitter_contact": "contact@example.com", + "system_name": "sys_h200_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8562Y+", + "host_processors_per_node": 2, + "host_processor_core_count": 32, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H200-SXM-141GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "141 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 1.0 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/config.yaml new file mode 100644 index 000000000..7db6216f5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/config.yaml @@ -0,0 +1 @@ +concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/point.yaml new file mode 100644 index 000000000..9f4a04371 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/point.yaml @@ -0,0 +1,49 @@ +concurrency: 64 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 64 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 640 + requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/result_summary.json new file mode 100644 index 000000000..8463177f0 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 906, + "n_samples_completed": 906, + "n_samples_failed": 0, + "duration_ns": 2209290005442.0, + "ttft": { + "total": 134375863380.0, + "percentiles": { + "50": 148317730.0, + "95": 300972722.5, + "90": 270441724.0 + } + }, + "output_sequence_lengths": { + "total": 3400893.0, + "percentiles": {} + }, + "tpot": { + "total": 3668394000.0000005, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1539.3602, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/system_desc.json new file mode 100644 index 000000000..bd61b0f41 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/system_desc.json @@ -0,0 +1,62 @@ +{ + "submitter_org_names": "Submitter-I", + "submitter_contact": "contact@example.com", + "system_name": "sys_h200_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8562Y+", + "host_processors_per_node": 2, + "host_processor_core_count": 32, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H200-SXM-141GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "141 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.235336 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/config.yaml new file mode 100644 index 000000000..e1b56a7e8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/config.yaml @@ -0,0 +1 @@ +concurrency: 8 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/point.yaml new file mode 100644 index 000000000..ff5648af6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/point.yaml @@ -0,0 +1,49 @@ +concurrency: 8 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 512 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 8 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 80 + requests_issued: 80 diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/result_summary.json new file mode 100644 index 000000000..cac202f46 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 220, + "n_samples_completed": 220, + "n_samples_failed": 0, + "duration_ns": 1919705564758.0, + "ttft": { + "total": 28929776150.000004, + "percentiles": { + "50": 131498982.50000001, + "95": 139797791.85, + "90": 138138029.98 + } + }, + "output_sequence_lengths": { + "total": 867074.0, + "percentiles": {} + }, + "tpot": { + "total": 890780000.0000001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 451.6703, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/system_desc.json new file mode 100644 index 000000000..bb73e7d98 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/system_desc.json @@ -0,0 +1,62 @@ +{ + "submitter_org_names": "Submitter-I", + "submitter_contact": "contact@example.com", + "system_name": "sys_h200_8gpu", + "system_category": "datacenter", + "max_supported_concurrency": 512, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "Intel Xeon Platinum 8562Y+", + "host_processors_per_node": 2, + "host_processor_core_count": 32, + "host_memory_capacity": "2 TiB", + "host_networking": "InfiniBand NDR", + "host_storage_capacity": "15 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H200-SXM-141GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "141 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Air-cooled", + "tps_utilization": 0.069051 +} diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/system_power.json b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/system_power.json new file mode 100644 index 000000000..dec21d8a6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/system_power.json @@ -0,0 +1,44 @@ +{ + "system_desc_id": "sys_h200_8gpu", + "cooling": "air", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": { + "model": "Intel Xeon Platinum 8562Y+", + "count_per_node": 2, + "tdp_per_unit": { + "value_w": 350, + "source_type": "vendor_spec", + "source": "https://example.com/cpu-spec" + } + }, + "accelerator": { + "model": "NVIDIA H200-SXM-141GB", + "count_per_node": 8, + "tdp_per_unit": { + "value_w": 700, + "source_type": "vendor_spec", + "source": "https://example.com/accelerator-spec" + } + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": { + "value_w": 3500, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + } + } + ], + "scale_out": { + "present": false + } +} diff --git a/tests/fixtures/validation/submissions/sub_i/src/tensorrt-llm/README.md b/tests/fixtures/validation/submissions/sub_i/src/tensorrt-llm/README.md new file mode 100644 index 000000000..ab1543fd0 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_i/src/tensorrt-llm/README.md @@ -0,0 +1,3 @@ +# tensorrt-llm + +Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_j/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_j/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/config.yaml new file mode 100644 index 000000000..b8581faa1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/config.yaml @@ -0,0 +1 @@ +concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/point.yaml new file mode 100644 index 000000000..b786427c8 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/point.yaml @@ -0,0 +1,49 @@ +concurrency: 1024 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 16384 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1024 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 10240 + requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/result_summary.json new file mode 100644 index 000000000..5fa3c1f9e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 8776, + "n_samples_completed": 8776, + "n_samples_failed": 0, + "duration_ns": 323142428984.0, + "ttft": { + "total": 3105883110512.0, + "percentiles": { + "50": 353906462.0, + "95": 1762244150.0, + "90": 1480576612.4 + } + }, + "output_sequence_lengths": { + "total": 34315466.0, + "percentiles": {} + }, + "tpot": { + "total": 35534024000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 106193.0063, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/system_desc.json new file mode 100644 index 000000000..1e1930b42 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-J", + "submitter_contact": "contact@example.com", + "system_name": "sys_gb300_72gpu", + "system_category": "datacenter", + "max_supported_concurrency": 16384, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", + "host_processors_per_node": 36, + "host_processor_core_count": 72, + "host_memory_capacity": "2.16 TB LPDDR5x", + "host_networking": "InfiniBand NDR400", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04 (aarch64)", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA GB300", + "accelerators_per_node": 72, + "accelerator_memory_capacity": "288 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 5.0" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 72, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.283152 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/config.yaml new file mode 100644 index 000000000..51457c3cf --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/config.yaml @@ -0,0 +1 @@ +concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/point.yaml new file mode 100644 index 000000000..5c1c63043 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/point.yaml @@ -0,0 +1,49 @@ +concurrency: 128 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 16384 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 128 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 1280 + requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/result_summary.json new file mode 100644 index 000000000..e0332ec71 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 8776, + "n_samples_completed": 8776, + "n_samples_failed": 0, + "duration_ns": 1335987189179.0002, + "ttft": { + "total": 2152528849644.0, + "percentiles": { + "50": 245274481.5, + "95": 334071958.0, + "90": 316312462.7 + } + }, + "output_sequence_lengths": { + "total": 34063493.0, + "percentiles": {} + }, + "tpot": { + "total": 35534024000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 25496.871, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/system_desc.json new file mode 100644 index 000000000..eaf0a9158 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-J", + "submitter_contact": "contact@example.com", + "system_name": "sys_gb300_72gpu", + "system_category": "datacenter", + "max_supported_concurrency": 16384, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", + "host_processors_per_node": 36, + "host_processor_core_count": 72, + "host_memory_capacity": "2.16 TB LPDDR5x", + "host_networking": "InfiniBand NDR400", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04 (aarch64)", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA GB300", + "accelerators_per_node": 72, + "accelerator_memory_capacity": "288 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 5.0" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 72, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.067985 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/config.yaml new file mode 100644 index 000000000..74340a912 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/config.yaml @@ -0,0 +1 @@ +concurrency: 16384 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/point.yaml new file mode 100644 index 000000000..0c84640b6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/point.yaml @@ -0,0 +1,50 @@ +concurrency: 16384 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 16384 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +offline: elected +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 16384 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 163840 + requests_issued: 163840 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/result_summary.json new file mode 100644 index 000000000..4a036b18c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 417284, + "n_samples_completed": 417284, + "n_samples_failed": 0, + "duration_ns": 4165777685634.0, + "ttft": { + "total": 178378755811144.0, + "percentiles": { + "50": 427475666.0, + "95": 8529496162.349946, + "90": 6909092063.079957 + } + }, + "output_sequence_lengths": { + "total": 1562330261.0, + "percentiles": {} + }, + "tpot": { + "total": 1689582916000.0002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 375039.2793, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/system_desc.json new file mode 100644 index 000000000..1ad0109a1 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-J", + "submitter_contact": "contact@example.com", + "system_name": "sys_gb300_72gpu", + "system_category": "datacenter", + "max_supported_concurrency": 16384, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", + "host_processors_per_node": 36, + "host_processor_core_count": 72, + "host_memory_capacity": "2.16 TB LPDDR5x", + "host_networking": "InfiniBand NDR400", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04 (aarch64)", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA GB300", + "accelerators_per_node": 72, + "accelerator_memory_capacity": "288 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 5.0" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 72, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 1.0 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/config.yaml new file mode 100644 index 000000000..6b53d1bba --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/config.yaml @@ -0,0 +1 @@ +concurrency: 2048 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/point.yaml new file mode 100644 index 000000000..843d67b73 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/point.yaml @@ -0,0 +1,49 @@ +concurrency: 2048 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 16384 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 2048 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 20480 + requests_issued: 20480 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/result_summary.json new file mode 100644 index 000000000..dabf2927b --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 8776, + "n_samples_completed": 8776, + "n_samples_failed": 0, + "duration_ns": 247915387632.0, + "ttft": { + "total": 4262954461920.0, + "percentiles": { + "50": 485751420.0, + "95": 3795499436.5, + "90": 3133549833.2000003 + } + }, + "output_sequence_lengths": { + "total": 33975939.0, + "percentiles": {} + }, + "tpot": { + "total": 35534024000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 137046.5114, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/system_desc.json new file mode 100644 index 000000000..3dc33743a --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-J", + "submitter_contact": "contact@example.com", + "system_name": "sys_gb300_72gpu", + "system_category": "datacenter", + "max_supported_concurrency": 16384, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", + "host_processors_per_node": 36, + "host_processor_core_count": 72, + "host_memory_capacity": "2.16 TB LPDDR5x", + "host_networking": "InfiniBand NDR400", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04 (aarch64)", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA GB300", + "accelerators_per_node": 72, + "accelerator_memory_capacity": "288 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 5.0" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 72, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.365419 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/config.yaml new file mode 100644 index 000000000..f928653b5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/config.yaml @@ -0,0 +1 @@ +concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/point.yaml new file mode 100644 index 000000000..83661b237 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/point.yaml @@ -0,0 +1,49 @@ +concurrency: 256 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 16384 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 256 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 2560 + requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/result_summary.json new file mode 100644 index 000000000..7b8c8f2b2 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 8776, + "n_samples_completed": 8776, + "n_samples_failed": 0, + "duration_ns": 782307149059.0001, + "ttft": { + "total": 2172863837720.0, + "percentiles": { + "50": 247591595.0, + "95": 505510358.75, + "90": 453926606.0 + } + }, + "output_sequence_lengths": { + "total": 33878035.0, + "percentiles": {} + }, + "tpot": { + "total": 35534024000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 43305.2862, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/system_desc.json new file mode 100644 index 000000000..7f3422cff --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-J", + "submitter_contact": "contact@example.com", + "system_name": "sys_gb300_72gpu", + "system_category": "datacenter", + "max_supported_concurrency": 16384, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", + "host_processors_per_node": 36, + "host_processor_core_count": 72, + "host_memory_capacity": "2.16 TB LPDDR5x", + "host_networking": "InfiniBand NDR400", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04 (aarch64)", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA GB300", + "accelerators_per_node": 72, + "accelerator_memory_capacity": "288 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 5.0" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 72, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.115469 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/config.yaml new file mode 100644 index 000000000..b03f20973 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/config.yaml @@ -0,0 +1 @@ +concurrency: 32 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/point.yaml new file mode 100644 index 000000000..54d1f355f --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/point.yaml @@ -0,0 +1,49 @@ +concurrency: 32 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 16384 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 32 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 320 + requests_issued: 320 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/result_summary.json new file mode 100644 index 000000000..2bfc7a8d6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 8776, + "n_samples_completed": 8776, + "n_samples_failed": 0, + "duration_ns": 5076457317773.0, + "ttft": { + "total": 2109229364492.0, + "percentiles": { + "50": 240340629.5, + "95": 315733786.5, + "90": 300655155.1 + } + }, + "output_sequence_lengths": { + "total": 33926283.0, + "percentiles": {} + }, + "tpot": { + "total": 35534024000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 6683.0628, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/system_desc.json new file mode 100644 index 000000000..f3715be50 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-J", + "submitter_contact": "contact@example.com", + "system_name": "sys_gb300_72gpu", + "system_category": "datacenter", + "max_supported_concurrency": 16384, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", + "host_processors_per_node": 36, + "host_processor_core_count": 72, + "host_memory_capacity": "2.16 TB LPDDR5x", + "host_networking": "InfiniBand NDR400", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04 (aarch64)", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA GB300", + "accelerators_per_node": 72, + "accelerator_memory_capacity": "288 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 5.0" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 72, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.01782 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/config.yaml new file mode 100644 index 000000000..d3e2cf53c --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/config.yaml @@ -0,0 +1 @@ +concurrency: 4096 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/point.yaml new file mode 100644 index 000000000..136e3273f --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/point.yaml @@ -0,0 +1,49 @@ +concurrency: 4096 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 16384 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 4096 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 40960 + requests_issued: 40960 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/result_summary.json new file mode 100644 index 000000000..f066438ca --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 8776, + "n_samples_completed": 8776, + "n_samples_failed": 0, + "duration_ns": 196988273307.0, + "ttft": { + "total": 45268755452096.0, + "percentiles": { + "50": 5158244696.0, + "95": 6765218642.5, + "90": 6443823853.2 + } + }, + "output_sequence_lengths": { + "total": 34013009.0, + "percentiles": {} + }, + "tpot": { + "total": 35534024000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 172665.1461, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/system_desc.json new file mode 100644 index 000000000..94ba28ba6 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-J", + "submitter_contact": "contact@example.com", + "system_name": "sys_gb300_72gpu", + "system_category": "datacenter", + "max_supported_concurrency": 16384, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", + "host_processors_per_node": 36, + "host_processor_core_count": 72, + "host_memory_capacity": "2.16 TB LPDDR5x", + "host_networking": "InfiniBand NDR400", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04 (aarch64)", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA GB300", + "accelerators_per_node": 72, + "accelerator_memory_capacity": "288 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 5.0" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 72, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.460392 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/config.yaml new file mode 100644 index 000000000..846832e5e --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/config.yaml @@ -0,0 +1 @@ +concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/point.yaml new file mode 100644 index 000000000..da7ab6472 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/point.yaml @@ -0,0 +1,49 @@ +concurrency: 512 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 16384 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 512 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 5120 + requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/result_summary.json new file mode 100644 index 000000000..65c5ee05d --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 8776, + "n_samples_completed": 8776, + "n_samples_failed": 0, + "duration_ns": 495036806948.0, + "ttft": { + "total": 2712508796676.0, + "percentiles": { + "50": 309082588.5, + "95": 569378711.0, + "90": 517319486.5 + } + }, + "output_sequence_lengths": { + "total": 34233018.0, + "percentiles": {} + }, + "tpot": { + "total": 35534024000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 69152.4701, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/system_desc.json new file mode 100644 index 000000000..43a2259bc --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-J", + "submitter_contact": "contact@example.com", + "system_name": "sys_gb300_72gpu", + "system_category": "datacenter", + "max_supported_concurrency": 16384, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", + "host_processors_per_node": 36, + "host_processor_core_count": 72, + "host_memory_capacity": "2.16 TB LPDDR5x", + "host_networking": "InfiniBand NDR400", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04 (aarch64)", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA GB300", + "accelerators_per_node": 72, + "accelerator_memory_capacity": "288 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 5.0" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 72, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.184387 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/config.yaml new file mode 100644 index 000000000..7db6216f5 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/config.yaml @@ -0,0 +1 @@ +concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/point.yaml new file mode 100644 index 000000000..6327faa28 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/point.yaml @@ -0,0 +1,49 @@ +concurrency: 64 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 16384 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 64 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 640 + requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/result_summary.json new file mode 100644 index 000000000..051a5ab73 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 8776, + "n_samples_completed": 8776, + "n_samples_failed": 0, + "duration_ns": 2602571018770.0, + "ttft": { + "total": 2027031584516.0, + "percentiles": { + "50": 230974428.5, + "95": 307411728.5, + "90": 292124268.5 + } + }, + "output_sequence_lengths": { + "total": 34185331.0, + "percentiles": {} + }, + "tpot": { + "total": 35534024000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 13135.2154, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/system_desc.json new file mode 100644 index 000000000..37588a256 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-J", + "submitter_contact": "contact@example.com", + "system_name": "sys_gb300_72gpu", + "system_category": "datacenter", + "max_supported_concurrency": 16384, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", + "host_processors_per_node": 36, + "host_processor_core_count": 72, + "host_memory_capacity": "2.16 TB LPDDR5x", + "host_networking": "InfiniBand NDR400", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04 (aarch64)", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA GB300", + "accelerators_per_node": 72, + "accelerator_memory_capacity": "288 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 5.0" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 72, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.035024 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/accuracy_results.json new file mode 100644 index 000000000..0a70c3eb7 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/accuracy_results.json @@ -0,0 +1,11 @@ +{ + "math500": { + "dataset_name": "math500", + "num_samples": 4388, + "score": { + "exact_match": 81.3582 + }, + "extractor": "BoxedAnswerExtractor", + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/config.yaml new file mode 100644 index 000000000..69f442619 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/config.yaml @@ -0,0 +1 @@ +concurrency: 8192 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/point.yaml new file mode 100644 index 000000000..98f6e7483 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/point.yaml @@ -0,0 +1,49 @@ +concurrency: 8192 +dataset: mlperf-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 16384 +model_name: deepseek-r1 +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 8192 + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 81920 + requests_issued: 81920 diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/result_summary.json new file mode 100644 index 000000000..7d989d711 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 417284, + "n_samples_completed": 417284, + "n_samples_failed": 0, + "duration_ns": 6231386322474.0, + "ttft": { + "total": 165929879676176.0, + "percentiles": { + "50": 397642564.0, + "95": 641731587.85, + "90": 592913783.08 + } + }, + "output_sequence_lengths": { + "total": 1564302823.0, + "percentiles": {} + }, + "tpot": { + "total": 1689582916000.0002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 251036.0844, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/system_desc.json new file mode 100644 index 000000000..a23ab44fc --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/system_desc.json @@ -0,0 +1,61 @@ +{ + "submitter_org_names": "Submitter-J", + "submitter_contact": "contact@example.com", + "system_name": "sys_gb300_72gpu", + "system_category": "datacenter", + "max_supported_concurrency": 16384, + "serving_framework": "TensorRT-LLM", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", + "host_processors_per_node": 36, + "host_processor_core_count": 72, + "host_memory_capacity": "2.16 TB LPDDR5x", + "host_networking": "InfiniBand NDR400", + "host_storage_capacity": "30 TB", + "host_storage_type": "NVMe SSD", + "operating_system": "Ubuntu 22.04 (aarch64)", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA GB300", + "accelerators_per_node": 72, + "accelerator_memory_capacity": "288 GB HBM3e", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 5.0" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "deepseek-r1", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 72, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "cooling": "Liquid-cooled", + "tps_utilization": 0.669359 +} diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/system_power.json b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/system_power.json new file mode 100644 index 000000000..0287518fc --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/system_power.json @@ -0,0 +1,44 @@ +{ + "system_desc_id": "sys_gb300_72gpu", + "cooling": "liquid", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": { + "model": "NVIDIA Grace CPU (Armv9)", + "count_per_node": 36, + "tdp_per_unit": { + "value_w": 350, + "source_type": "vendor_spec", + "source": "https://example.com/cpu-spec" + } + }, + "accelerator": { + "model": "NVIDIA GB300", + "count_per_node": 72, + "tdp_per_unit": { + "value_w": 700, + "source_type": "vendor_spec", + "source": "https://example.com/accelerator-spec" + } + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": { + "value_w": 3500, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + } + } + ], + "scale_out": { + "present": false + } +} diff --git a/tests/fixtures/validation/submissions/sub_j/src/tensorrt-llm/README.md b/tests/fixtures/validation/submissions/sub_j/src/tensorrt-llm/README.md new file mode 100644 index 000000000..ab1543fd0 --- /dev/null +++ b/tests/fixtures/validation/submissions/sub_j/src/tensorrt-llm/README.md @@ -0,0 +1,3 @@ +# tensorrt-llm + +Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/valid_standardized/docs/.gitkeep b/tests/fixtures/validation/submissions/valid_standardized/docs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/valid_standardized/docs/calibration.adoc b/tests/fixtures/validation/submissions/valid_standardized/docs/calibration.adoc new file mode 100644 index 000000000..cc918cad0 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/docs/calibration.adoc @@ -0,0 +1,5 @@ += Weight Transformation Documentation + +== Quantization + +Applied INT8 post-training quantization using GPTQ with 512-sample calibration dataset drawn from the performance dataset. No gradient updates applied. Calibration dataset disclosed in full in calibration_data/. diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/accuracy_results.json new file mode 100644 index 000000000..b37d348d9 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/accuracy_results.json @@ -0,0 +1,13 @@ +{ + "llm-perf-dataset-v1": { + "dataset_name": "llm-perf-dataset-v1", + "num_samples": 13368, + "extractor": "RougeExtractor", + "score": { + "rouge1": "45.12", + "rouge2": "22.01", + "rougeL": "30.45" + }, + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/config.yaml new file mode 100644 index 000000000..ad1450850 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/config.yaml @@ -0,0 +1 @@ +concurrency: 1000 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/point.yaml new file mode 100644 index 000000000..7085286ce --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/point.yaml @@ -0,0 +1,50 @@ +concurrency: 1000 +dataset: llm-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1000 +model_name: llama3_1-8b +model_notes: "" +model_precision: FP16 +offline: elected +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 1000 + data_source: llm-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 10000 + requests_issued: 10000 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/result_summary.json new file mode 100644 index 000000000..793001ed3 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 22000, + "n_samples_completed": 22000, + "n_samples_failed": 0, + "duration_ns": 1200000000000, + "ttft": { + "total": 9259800000000.0, + "percentiles": { + "50": 420900000.0, + "95": 890500000.0, + "90": 796580000.0 + } + }, + "output_sequence_lengths": { + "total": 5400000.0, + "percentiles": {} + }, + "tpot": { + "total": 89078000000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 4500.0, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/system_desc.json new file mode 100644 index 000000000..b0dc4f6e2 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/system_desc.json @@ -0,0 +1,63 @@ +{ + "submitter_org_names": "ACME Corp", + "submitter_contact": "contact@example.com", + "system_name": "ACME H100x8 Node", + "system_category": "datacenter", + "max_supported_concurrency": 1000, + "serving_framework": "vLLM 0.4.2", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "20 TB", + "host_storage_type": "NVMe SSD", + "cooling": "Air-cooled", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100 SXM5 80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3.1-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "model_name": "llama3.1-8b", + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "tps_utilization": 0.978261 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/accuracy_results.json new file mode 100644 index 000000000..b37d348d9 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/accuracy_results.json @@ -0,0 +1,13 @@ +{ + "llm-perf-dataset-v1": { + "dataset_name": "llm-perf-dataset-v1", + "num_samples": 13368, + "extractor": "RougeExtractor", + "score": { + "rouge1": "45.12", + "rouge2": "22.01", + "rougeL": "30.45" + }, + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/config.yaml new file mode 100644 index 000000000..f9afe658a --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/config.yaml @@ -0,0 +1 @@ +concurrency: 16 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/point.yaml new file mode 100644 index 000000000..e19f1fe77 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/point.yaml @@ -0,0 +1,49 @@ +concurrency: 16 +dataset: llm-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1000 +model_name: llama3_1-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 600000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 600.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 16 + data_source: llm-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 160 + requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/result_summary.json new file mode 100644 index 000000000..e6e2bc2a1 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 3000, + "n_samples_completed": 3000, + "n_samples_failed": 0, + "duration_ns": 600000000000, + "ttft": { + "total": 135600000000.0, + "percentiles": { + "50": 45200000.0, + "95": 89100000.0, + "90": 80320000.0 + } + }, + "output_sequence_lengths": { + "total": 480000.0, + "percentiles": {} + }, + "tpot": { + "total": 12147000000.000002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 800.0, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/system_desc.json new file mode 100644 index 000000000..cfa21d246 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/system_desc.json @@ -0,0 +1,63 @@ +{ + "submitter_org_names": "ACME Corp", + "submitter_contact": "contact@example.com", + "system_name": "ACME H100x8 Node", + "system_category": "datacenter", + "max_supported_concurrency": 1000, + "serving_framework": "vLLM 0.4.2", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "20 TB", + "host_storage_type": "NVMe SSD", + "cooling": "Air-cooled", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100 SXM5 80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3.1-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "model_name": "llama3.1-8b", + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "tps_utilization": 0.173913 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/accuracy_results.json new file mode 100644 index 000000000..b37d348d9 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/accuracy_results.json @@ -0,0 +1,13 @@ +{ + "llm-perf-dataset-v1": { + "dataset_name": "llm-perf-dataset-v1", + "num_samples": 13368, + "extractor": "RougeExtractor", + "score": { + "rouge1": "45.12", + "rouge2": "22.01", + "rougeL": "30.45" + }, + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/config.yaml new file mode 100644 index 000000000..1a6f2cb8e --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/config.yaml @@ -0,0 +1 @@ +concurrency: 38 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/point.yaml new file mode 100644 index 000000000..39fd6248b --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/point.yaml @@ -0,0 +1,49 @@ +concurrency: 20 +dataset: llm-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1000 +model_name: llama3_1-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 20 + data_source: llm-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 200 + requests_issued: 200 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/result_summary.json new file mode 100644 index 000000000..28f9c21c7 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 8000, + "n_samples_completed": 8000, + "n_samples_failed": 0, + "duration_ns": 1200000000000, + "ttft": { + "total": 418400000000.0, + "percentiles": { + "50": 52300000.0, + "95": 110400000.0, + "90": 98780000.0 + } + }, + "output_sequence_lengths": { + "total": 1800000.0, + "percentiles": {} + }, + "tpot": { + "total": 32392000000.000004, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 1500.0, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/system_desc.json new file mode 100644 index 000000000..54c2146a6 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/system_desc.json @@ -0,0 +1,63 @@ +{ + "submitter_org_names": "ACME Corp", + "submitter_contact": "contact@example.com", + "system_name": "ACME H100x8 Node", + "system_category": "datacenter", + "max_supported_concurrency": 1000, + "serving_framework": "vLLM 0.4.2", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "20 TB", + "host_storage_type": "NVMe SSD", + "cooling": "Air-cooled", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100 SXM5 80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3.1-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "model_name": "llama3.1-8b", + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "tps_utilization": 0.326087 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/accuracy_results.json new file mode 100644 index 000000000..b37d348d9 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/accuracy_results.json @@ -0,0 +1,13 @@ +{ + "llm-perf-dataset-v1": { + "dataset_name": "llm-perf-dataset-v1", + "num_samples": 13368, + "extractor": "RougeExtractor", + "score": { + "rouge1": "45.12", + "rouge2": "22.01", + "rougeL": "30.45" + }, + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/config.yaml new file mode 100644 index 000000000..f928653b5 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/config.yaml @@ -0,0 +1 @@ +concurrency: 256 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/point.yaml new file mode 100644 index 000000000..d85e69bae --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/point.yaml @@ -0,0 +1,49 @@ +concurrency: 256 +dataset: llm-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1000 +model_name: llama3_1-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 256 + data_source: llm-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 2560 + requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/result_summary.json new file mode 100644 index 000000000..b9aa245f8 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 20000, + "n_samples_completed": 20000, + "n_samples_failed": 0, + "duration_ns": 1200000000000, + "ttft": { + "total": 2050000000000.0, + "percentiles": { + "50": 102500000.0, + "95": 220800000.0, + "90": 197140000.0 + } + }, + "output_sequence_lengths": { + "total": 4800000.0, + "percentiles": {} + }, + "tpot": { + "total": 80980000000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 4000.0, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/system_desc.json new file mode 100644 index 000000000..01ab8325f --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/system_desc.json @@ -0,0 +1,63 @@ +{ + "submitter_org_names": "ACME Corp", + "submitter_contact": "contact@example.com", + "system_name": "ACME H100x8 Node", + "system_category": "datacenter", + "max_supported_concurrency": 1000, + "serving_framework": "vLLM 0.4.2", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "20 TB", + "host_storage_type": "NVMe SSD", + "cooling": "Air-cooled", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100 SXM5 80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3.1-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "model_name": "llama3.1-8b", + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "tps_utilization": 0.869565 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/accuracy_results.json new file mode 100644 index 000000000..b37d348d9 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/accuracy_results.json @@ -0,0 +1,13 @@ +{ + "llm-perf-dataset-v1": { + "dataset_name": "llm-perf-dataset-v1", + "num_samples": 13368, + "extractor": "RougeExtractor", + "score": { + "rouge1": "45.12", + "rouge2": "22.01", + "rougeL": "30.45" + }, + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/config.yaml new file mode 100644 index 000000000..846832e5e --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/config.yaml @@ -0,0 +1 @@ +concurrency: 512 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/point.yaml new file mode 100644 index 000000000..f6c0decb8 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/point.yaml @@ -0,0 +1,49 @@ +concurrency: 512 +dataset: llm-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1000 +model_name: llama3_1-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 512 + data_source: llm-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 5120 + requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/result_summary.json new file mode 100644 index 000000000..5c94bee96 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 22000, + "n_samples_completed": 22000, + "n_samples_failed": 0, + "duration_ns": 1200000000000, + "ttft": { + "total": 3966600000000.0, + "percentiles": { + "50": 180300000.0, + "95": 410200000.0, + "90": 364220000.0 + } + }, + "output_sequence_lengths": { + "total": 5400000.0, + "percentiles": {} + }, + "tpot": { + "total": 89078000000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 4500.0, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/system_desc.json new file mode 100644 index 000000000..b0dc4f6e2 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/system_desc.json @@ -0,0 +1,63 @@ +{ + "submitter_org_names": "ACME Corp", + "submitter_contact": "contact@example.com", + "system_name": "ACME H100x8 Node", + "system_category": "datacenter", + "max_supported_concurrency": 1000, + "serving_framework": "vLLM 0.4.2", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "20 TB", + "host_storage_type": "NVMe SSD", + "cooling": "Air-cooled", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100 SXM5 80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3.1-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "model_name": "llama3.1-8b", + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "tps_utilization": 0.978261 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/accuracy_results.json new file mode 100644 index 000000000..b37d348d9 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/accuracy_results.json @@ -0,0 +1,13 @@ +{ + "llm-perf-dataset-v1": { + "dataset_name": "llm-perf-dataset-v1", + "num_samples": 13368, + "extractor": "RougeExtractor", + "score": { + "rouge1": "45.12", + "rouge2": "22.01", + "rougeL": "30.45" + }, + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/config.yaml new file mode 100644 index 000000000..240492592 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/config.yaml @@ -0,0 +1 @@ +concurrency: 768 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/point.yaml new file mode 100644 index 000000000..c120f6e32 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/point.yaml @@ -0,0 +1,49 @@ +concurrency: 768 +dataset: llm-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1000 +model_name: llama3_1-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 768 + data_source: llm-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 7680 + requests_issued: 7680 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/result_summary.json new file mode 100644 index 000000000..c44d18beb --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 23000, + "n_samples_completed": 23000, + "n_samples_failed": 0, + "duration_ns": 1200000000000, + "ttft": { + "total": 6801100000000.0, + "percentiles": { + "50": 295700000.0, + "95": 620100000.0, + "90": 555220000.0 + } + }, + "output_sequence_lengths": { + "total": 5520000.0, + "percentiles": {} + }, + "tpot": { + "total": 93127000000.00002, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 4600.0, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/system_desc.json new file mode 100644 index 000000000..8a9e0fd70 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/system_desc.json @@ -0,0 +1,63 @@ +{ + "submitter_org_names": "ACME Corp", + "submitter_contact": "contact@example.com", + "system_name": "ACME H100x8 Node", + "system_category": "datacenter", + "max_supported_concurrency": 1000, + "serving_framework": "vLLM 0.4.2", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "20 TB", + "host_storage_type": "NVMe SSD", + "cooling": "Air-cooled", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100 SXM5 80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3.1-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "model_name": "llama3.1-8b", + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "tps_utilization": 1.0 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/accuracy_results.json new file mode 100644 index 000000000..b37d348d9 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/accuracy_results.json @@ -0,0 +1,13 @@ +{ + "llm-perf-dataset-v1": { + "dataset_name": "llm-perf-dataset-v1", + "num_samples": 13368, + "extractor": "RougeExtractor", + "score": { + "rouge1": "45.12", + "rouge2": "22.01", + "rougeL": "30.45" + }, + "n_repeats": 1 + } +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/config.yaml new file mode 100644 index 000000000..cf16abd9b --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/config.yaml @@ -0,0 +1 @@ +concurrency: 88 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/point.yaml new file mode 100644 index 000000000..34bb7c7fc --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/point.yaml @@ -0,0 +1,49 @@ +concurrency: 88 +dataset: llm-perf-dataset-v1 +dataset_link: https://example.com/dataset +dataset_name: CNN/DailyMail +dataset_type: Performance +division: Standardized +link_to_model: https://example.com/model +link_to_model_transformation: https://example.com/quantization +max_supported_concurrency: 1000 +model_name: llama3_1-8b +model_notes: "" +model_precision: FP16 +runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true +seed_set: A +shared_docs: docs +shared_src: src +steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 +target_cohort: 2026-10-C1 +warmup: + concurrency: 88 + data_source: llm-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + requests_completed: 880 + requests_issued: 880 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/result_summary.json new file mode 100644 index 000000000..aee0f8a03 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/result_summary.json @@ -0,0 +1,29 @@ +{ + "n_samples_issued": 12000, + "n_samples_completed": 12000, + "n_samples_failed": 0, + "duration_ns": 1200000000000, + "ttft": { + "total": 817200000000.0, + "percentiles": { + "50": 68100000.0, + "95": 145200000.0, + "90": 129780000.0 + } + }, + "output_sequence_lengths": { + "total": 3000000.0, + "percentiles": {} + }, + "tpot": { + "total": 48588000000.00001, + "percentiles": { + "50": 4000000.0, + "90": 5500000.0, + "95": 6000000.0, + "99": 6500000.0 + } + }, + "system_tps": 2500.0, + "tps_per_user": 181.8182 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/system_desc.json new file mode 100644 index 000000000..1cfedfcba --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/system_desc.json @@ -0,0 +1,63 @@ +{ + "submitter_org_names": "ACME Corp", + "submitter_contact": "contact@example.com", + "system_name": "ACME H100x8 Node", + "system_category": "datacenter", + "max_supported_concurrency": 1000, + "serving_framework": "vLLM 0.4.2", + "node_types": [ + { + "system_node_ensemble_id": 0, + "number_of_nodes": 1, + "host_processor_model_name": "AMD EPYC 9654", + "host_processors_per_node": 2, + "host_processor_core_count": 96, + "host_memory_capacity": "768 GB", + "host_networking": "InfiniBand HDR", + "host_storage_capacity": "20 TB", + "host_storage_type": "NVMe SSD", + "cooling": "Air-cooled", + "operating_system": "Ubuntu 22.04", + "host_memory_configuration": "8x 64GB DDR5", + "host_network_card_count": "1x NIC", + "driver": "1.0", + "filesystem": "ext4", + "accelerator_info": [ + { + "accelerator_model_name": "NVIDIA H100 SXM5 80GB", + "accelerators_per_node": 8, + "accelerator_memory_capacity": "80 GB HBM3", + "accelerator_memory_type": "HBM3e", + "accelerator_interconnect": "NVLink 4.0 900 GB/s", + "accelerator_host_interconnect": "PCIe Gen5" + } + ] + } + ], + "division": "Standardized", + "model_precision": "FP16", + "link_to_model": "https://example.com/model", + "dataset_id": "cnn_dailymail", + "dataset_name": "CNN/DailyMail", + "input_token_average": 870.0, + "output_token_average": 128.0, + "dataset_type": "performance", + "dataset_link": "https://example.com/dataset", + "measured_accuracy_score": "38.7", + "system_size": "1 node", + "model_id": "llama3.1-8b", + "system_node_ensemble_count": 1, + "system_node_ensemble_total": 1, + "model_name": "llama3.1-8b", + "publication_status": "Available", + "node_config": "8x H100 test node", + "config_summary": { + "disaggregated": null, + "expert_parallel": 1, + "tensor_parallel": 8, + "pipeline_parallel": 1, + "data_parallel": 1, + "batch": null + }, + "tps_utilization": 0.543478 +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/system_power.json b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/system_power.json new file mode 100644 index 000000000..2766647e6 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/system_power.json @@ -0,0 +1,44 @@ +{ + "system_desc_id": "acme_h100x8_001", + "cooling": "air", + "node_sets": [ + { + "node_set_id": 0, + "system_node_ensemble_id": 0, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": { + "model": "AMD EPYC 9654", + "count_per_node": 2, + "tdp_per_unit": { + "value_w": 350, + "source_type": "vendor_spec", + "source": "https://example.com/cpu-spec" + } + }, + "accelerator": { + "model": "NVIDIA H100 SXM5 80GB", + "count_per_node": 8, + "tdp_per_unit": { + "value_w": 700, + "source_type": "vendor_spec", + "source": "https://example.com/accelerator-spec" + } + }, + "scale_up_network": { + "method": "declared_tdp", + "switch_count": 1, + "tdp_per_switch": { + "value_w": 3500, + "source_type": "vendor_spec", + "source": "https://example.com/switch-spec" + } + } + } + } + ], + "scale_out": { + "present": false + } +} diff --git a/tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/.gitkeep b/tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/README.md b/tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/README.md new file mode 100644 index 000000000..45692c1a3 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/README.md @@ -0,0 +1,3 @@ +# llama3_1-8b + +How to build the SUT and reproduce a Pareto point. diff --git a/tests/fixtures/validation/submissions/valid_standardized/src/vllm/README.md b/tests/fixtures/validation/submissions/valid_standardized/src/vllm/README.md new file mode 100644 index 000000000..be3edfb30 --- /dev/null +++ b/tests/fixtures/validation/submissions/valid_standardized/src/vllm/README.md @@ -0,0 +1,3 @@ +# vllm + +Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/unit/validation/test_artifacts.py b/tests/unit/validation/test_artifacts.py index 4770c350f..be6bd7856 100644 --- a/tests/unit/validation/test_artifacts.py +++ b/tests/unit/validation/test_artifacts.py @@ -12,8 +12,10 @@ from inference_endpoint.validation import ( bundled_policy_path, load_policy, + validate_submission, ) from inference_endpoint.validation.artifacts import PointArtifacts, load_artifacts +from inference_endpoint.validation.types import EvidenceKey pytestmark = pytest.mark.unit @@ -82,6 +84,29 @@ def test_reported_metrics_do_not_resolve_to_calculated_values(tmp_path, document assert omitted.get_summary("system_tps") is None +def test_malformed_config_blocks_dependent_check(tmp_path): + point = tmp_path / "results" / "system" / "gpt-oss-120b" / "r16" + point.mkdir(parents=True) + (point / "point.yaml").write_text( + json.dumps({"concurrency": "bad", "runtime_settings": {"runtime": {}}}) + ) + policy = load_policy(bundled_policy_path()) + artifacts = load_artifacts(tmp_path, policy) + parsed = artifacts.index.points[str(point)] + assert parsed.config is None + assert parsed.evidence.config.errors + assert EvidenceKey.POINT_CONFIG in parsed.context.invalid + report = validate_submission(tmp_path, policy=policy) + assert any( + result.rule == "point-config-valid" and not result.passed + for result in report.results + ) + assert any( + result.rule == "target-cohort" and result.key == "blocked" + for result in report.results + ) + + @pytest.mark.parametrize("standalone_accuracy", [False, True]) def test_point_input_files_are_read_once( tmp_path, documents, monkeypatch, standalone_accuracy diff --git a/tests/unit/validation/test_bindings.py b/tests/unit/validation/test_bindings.py new file mode 100644 index 000000000..c6b2498e2 --- /dev/null +++ b/tests/unit/validation/test_bindings.py @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Checkpoint matching and exact client revision approval.""" + +from dataclasses import replace + +import pytest + +from inference_endpoint.validation import bundled_policy_path, load_policy +from inference_endpoint.validation.artifacts import ( + ArtifactIndex, + Artifacts, + PointArtifacts, +) +from inference_endpoint.validation.bindings import ArtifactBinding +from inference_endpoint.validation.conditions import Context +from inference_endpoint.validation.evidence.loaders import ParsedArtifact +from inference_endpoint.validation.models import freeze +from inference_endpoint.validation.planner import PlannedCheck +from inference_endpoint.validation.types import Decision, Scope + +pytestmark = pytest.mark.unit + + +def update_evidence(point, field, **updates): + artifact = getattr(point.evidence, field) + data = artifact.value.model_dump( + mode="json", exclude_unset=True, exclude_computed_fields=True + ) + data.update(updates) + parsed = ParsedArtifact.from_json(data, type(artifact.value), point.path, field) + assert not parsed.errors + point.evidence = replace(point.evidence, **{field: parsed}) + + +def binding_check(tmp_path, rule_id): + policy = load_policy(bundled_policy_path()) + context = Context(id=str(tmp_path), scope=Scope.POINT, model_id="kimi-k3") + rule = next(rule for rule in policy.checks if rule.id == rule_id) + point = PointArtifacts.from_json( + tmp_path, + {"concurrency": 1, "runtime_settings": {"runtime": {}}}, + {"n_samples_completed": 1, "duration_ns": 1e9}, + {}, + ) + point.context = context + artifacts = Artifacts( + tmp_path, policy, index=ArtifactIndex(points={context.id: point}) + ) + return PlannedCheck(context, rule, Decision.READY, "Applicable"), artifacts, point + + +def test_checkpoint_requires_exact_model_repository_revision(tmp_path): + check, artifacts, point = binding_check(tmp_path, "approved-checkpoint") + entry = artifacts.policy.catalogs["approved_checkpoints"]["kimi-k3"][0] + update_evidence(point, "config", checkpoint=dict(entry)) + assert all(result.passed for result in ArtifactBinding()(check, artifacts)) + update_evidence(point, "config", checkpoint={**dict(entry), "revision": "0" * 40}) + assert any(not result.passed for result in ArtifactBinding()(check, artifacts)) + + +def test_empty_client_approval_list_cannot_pass(tmp_path): + check, artifacts, point = binding_check(tmp_path, "endpoints-client-sha") + update_evidence(point, "summary", git_sha="a" * 40) + results = ArtifactBinding()(check, artifacts) + assert results[0].key == "blocked" + assert not results[0].passed + + +@pytest.mark.parametrize( + ("revision", "passed"), + [("a" * 40, True), ("b" * 40, True), ("c" * 40, False)], +) +def test_client_revision_requires_exact_allowlist_match(tmp_path, revision, passed): + check, artifacts, point = binding_check(tmp_path, "endpoints-client-sha") + catalogs = dict(artifacts.policy.catalogs) + catalogs["approved_client_revisions"] = ["a" * 40, "b" * 40] + artifacts.policy = replace(artifacts.policy, catalogs=freeze(catalogs)) + update_evidence(point, "summary", git_sha=revision) + results = ArtifactBinding()(check, artifacts) + assert results[0].passed is passed + assert results[0].key != "blocked" + + +@pytest.mark.parametrize("revision", [None, "abc123", "A" * 40, "g" * 40]) +def test_client_revision_requires_full_lowercase_sha(tmp_path, revision): + check, artifacts, point = binding_check(tmp_path, "endpoints-client-sha") + update_evidence(point, "summary", git_sha=revision) + results = ArtifactBinding()(check, artifacts) + assert not results[0].passed + assert "full lowercase Git SHA-1" in results[0].message diff --git a/tests/unit/validation/test_engine.py b/tests/unit/validation/test_engine.py new file mode 100644 index 000000000..cb372e4bb --- /dev/null +++ b/tests/unit/validation/test_engine.py @@ -0,0 +1,323 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy edits change executed checks without modifying Python evaluators.""" + +import json +import shutil +from dataclasses import replace +from pathlib import Path + +import pytest +import yaml + +from inference_endpoint.validation import ( + Conditions, + bundled_policy_path, + load_policy, + validate_submission, +) +from inference_endpoint.validation.artifacts import load_artifacts +from inference_endpoint.validation.engine import execute_submission +from inference_endpoint.validation.evaluator_base import Evaluator +from inference_endpoint.validation.models import Override, freeze +from inference_endpoint.validation.types import EvidenceKey, Severity +from inference_endpoint.validation.vocabulary import CheckKind + +pytestmark = pytest.mark.unit +FIXTURE = ( + Path(__file__).resolve().parents[2] + / "fixtures/validation/submissions/valid_standardized" +) + + +def policy_only(*ids): + policy = load_policy(bundled_policy_path()) + return replace( + policy, checks=tuple(rule for rule in policy.checks if rule.id in ids) + ) + + +def test_dataset_catalog_edit_changes_count_gate(): + policy = policy_only("min-query-count") + original = execute_submission(FIXTURE, policy) + catalog = { + **policy.catalogs, + "datasets": {"llm-perf-dataset-v1": {"sample_count": 10**12}}, + } + stricter = execute_submission(FIXTURE, replace(policy, catalogs=freeze(catalog))) + assert original.passed + assert not stricter.passed + assert all(f.rule == "min-query-count" for f in stricter.results) + + +def test_duration_catalog_edit_changes_runtime_gate(): + policy = policy_only("region-basis", "point-duration") + original = execute_submission(FIXTURE, policy) + catalog = { + **policy.catalogs, + "steady_state": { + "minimum_duration_ms": dict.fromkeys( + policy.catalogs["steady_state"]["minimum_duration_ms"], 10**12 + ) + }, + } + stricter = execute_submission(FIXTURE, replace(policy, catalogs=freeze(catalog))) + assert original.passed + assert not stricter.passed + + +def test_new_yaml_rule_and_selection_override_execute(tmp_path): + policy_dir = tmp_path / "2026-10-C1" + shutil.copytree(bundled_policy_path(), policy_dir) + point_yaml = policy_dir / "point_checks.yaml" + document = yaml.safe_load(point_yaml.read_text()) + document["checks"] = { + "new-completed-floor": { + "kind": "comparison", + "left": {"kind": "field", "field": "result_summary.n_samples_completed"}, + "operator": "greater_than", + "right": {"kind": "constant", "value": 10**12}, + } + } + point_yaml.write_text(yaml.safe_dump(document)) + policy = load_policy(policy_dir) + rule = next(r for r in policy.checks if r.id == "new-completed-floor") + assert not execute_submission(FIXTURE, replace(policy, checks=(rule,))).passed + document["checks"]["new-completed-floor"]["applies_to"] = { + "load_pattern": ["agentic_inference"] + } + point_yaml.write_text(yaml.safe_dump(document)) + policy = load_policy(policy_dir) + rule = next(r for r in policy.checks if r.id == "new-completed-floor") + assert execute_submission(FIXTURE, replace(policy, checks=(rule,))).results == [] + + +def test_untrusted_nan_region_basis_blocks_without_crashing(tmp_path): + submission = tmp_path / "submission" + shutil.copytree(FIXTURE, submission) + for path in submission.glob("results/*/*/r*/system_desc.json"): + data = json.loads(path.read_text()) + data["max_supported_concurrency"] = float("nan") + path.write_text(json.dumps(data)) + report = execute_submission( + submission, policy_only("region-computation", "point-duration") + ) + assert not report.passed + + +def test_registry_rejects_duplicate_kinds(): + with pytest.raises(ValueError, match="Duplicate evaluator"): + + class Duplicate(Evaluator, kind=CheckKind.COMPARISON): + def __call__(self, check, artifacts): + return [] + + +def test_model_accuracy_catalog_changes_public_results(): + policy = policy_only("accuracy-gate") + profile = { + "accuracy": { + "kind": "single_turn", + "datasets": ["llm-perf-dataset-v1"], + "metrics": {"rouge1": {"reference": 1, "minimum_multiplier": 1}}, + } + } + catalogs = {**policy.catalogs, "models": {"llama3_1-8b": profile}} + policy = replace(policy, catalogs=freeze(catalogs)) + assert validate_submission(FIXTURE, policy=policy).passed + profile["accuracy"]["metrics"]["rouge1"]["reference"] = 10**12 + stricter = replace(policy, catalogs=freeze(catalogs)) + assert not validate_submission(FIXTURE, policy=stricter).passed + + +def test_disabled_cohort_override_prevents_execution(tmp_path): + policy_dir = tmp_path / "2026-10-C1" + shutil.copytree(bundled_policy_path(), policy_dir) + catalog_file = policy_dir / "catalog.yaml" + catalog = yaml.safe_load(catalog_file.read_text()) + catalog["enrollment"]["overrides"] = [ + { + "id": "count-disabled", + "rule": "min-query-count", + "reason": "Cohort count exemption", + "applies_to": {"models": ["llama3_1-8b"]}, + "enabled": False, + } + ] + catalog_file.write_text(yaml.safe_dump(catalog)) + policy = load_policy(policy_dir) + policy = replace( + policy, + checks=tuple(rule for rule in policy.checks if rule.id == "min-query-count"), + ) + assert validate_submission(FIXTURE, policy=policy).results == [] + + +def test_unregistered_evaluator_blocks(monkeypatch): + monkeypatch.delitem(Evaluator.registry, CheckKind.COMPARISON) + report = execute_submission(FIXTURE, policy_only("metric-consistency-duration")) + assert report.errors + assert all(f.key == "blocked" for f in report.errors) + + +def test_effective_override_changes_evaluator_threshold(): + policy = policy_only("point-cap") + override = Override( + id="smaller-cohort-cap", + rule="point-cap", + reason="Cohort point limit", + applies_to=Conditions(models=("llama3_1-8b",)), + enabled=True, + requirements=freeze({"maximum": 1}), + ) + assert validate_submission(FIXTURE, policy=policy).passed + overridden = validate_submission( + FIXTURE, policy=replace(policy, overrides=(override,)) + ) + assert not overridden.passed + assert all("allowed [0, 1]" in finding.message for finding in overridden.results) + + +def test_disabled_region_basis_does_not_publish_dependencies(): + policy = policy_only("region-basis") + exemption = Override( + id="disabled-basis", + rule="region-basis", + reason="Disabled derivation", + applies_to=Conditions(models=("llama3_1-8b",)), + enabled=False, + requirements=freeze({}), + ) + artifacts = load_artifacts(FIXTURE, replace(policy, overrides=(exemption,))) + assert not artifacts.derived.regions + assert all( + EvidenceKey.COMPUTED_REGIONS not in point.context.available + for point in artifacts.index.points.values() + ) + + +def test_region_basis_uses_selected_collection_members(tmp_path): + submission = tmp_path / "submission" + shutil.copytree(FIXTURE, submission) + path = next(submission.glob("results/*/*/r16/point.yaml")) + data = yaml.safe_load(path.read_text()) + data["offline"] = "dedicated" + path.write_text(yaml.safe_dump(data)) + policy = policy_only("region-basis") + rule = replace(policy.checks[0], applies_to=Conditions(offline=("none",))) + artifacts = load_artifacts(submission, replace(policy, checks=(rule,))) + assert next(iter(artifacts.derived.regions.values()))["low_latency"] == (1, 20) + + +@pytest.mark.parametrize("rule_id", ["region-placement", "offline-point-present"]) +def test_unknown_collection_operation_blocks(rule_id): + policy = policy_only("region-basis", rule_id) + rules = tuple( + replace( + rule, requirements=freeze({**rule.requirements, "operation": "unsupported"}) + ) + if rule.id == rule_id + else rule + for rule in policy.checks + ) + report = execute_submission(FIXTURE, replace(policy, checks=rules)) + findings = [f for f in report.results if f.rule == rule_id] + assert findings and all(f.key == "blocked" for f in findings) + + +def test_offline_uses_effective_numeric_concurrency_floor(tmp_path): + submission = tmp_path / "submission" + shutil.copytree(FIXTURE, submission) + path = next(submission.glob("results/*/*/r1000/point.yaml")) + data = yaml.safe_load(path.read_text()) + data["offline"] = "dedicated" + path.write_text(yaml.safe_dump(data)) + policy = policy_only("offline-ordering") + rule = replace( + policy.checks[0], + requirements=freeze( + {**policy.checks[0].requirements, "concurrency_floor": 10**12} + ), + ) + report = execute_submission(submission, replace(policy, checks=(rule,))) + assert report.results and report.results[0].severity.value == "warning" + assert "1000000000000" in report.results[0].message + + +def test_elected_offline_uses_effective_required_concurrency(): + policy = policy_only("offline-point-present") + rule = replace( + policy.checks[0], + requirements=freeze( + {**policy.checks[0].requirements, "elected_must_equal": 10**12} + ), + ) + assert not execute_submission(FIXTURE, replace(policy, checks=(rule,))).passed + + +def test_schema_errors_use_effective_warning_severity(tmp_path): + submission = tmp_path / "submission" + shutil.copytree(FIXTURE, submission) + path = next(submission.glob("results/*/*/r16/point.yaml")) + data = yaml.safe_load(path.read_text()) + data["concurrency"] = "invalid" + path.write_text(yaml.safe_dump(data)) + policy = policy_only("point-config-valid") + rule = replace(policy.checks[0], severity=Severity.WARNING) + report = execute_submission(submission, replace(policy, checks=(rule,))) + assert report.passed and report.warnings + assert all(f.severity.value == "warning" for f in report.results if f.path == path) + + +def test_conflicting_region_basis_partitions_block_explicitly(): + policy = policy_only("region-basis", "point-duration") + partition = Override( + id="different-clamp", + rule="region-basis", + reason="Partitioned region basis", + applies_to=Conditions(offline=("elected",)), + enabled=True, + requirements=freeze({"upper_clamp": 4}), + ) + report = execute_submission(FIXTURE, replace(policy, overrides=(partition,))) + assert any( + f.rule == "region-basis" and f.key == "blocked" and "Conflicting" in f.message + for f in report.results + ) + assert any( + f.rule == "point-duration" and f.key == "blocked" for f in report.results + ) + + +@pytest.mark.parametrize( + "option", ["seed_sets_path", "approved_sped_decode_heads_path"] +) +def test_api_rejects_catalog_overrides(option, tmp_path): + with pytest.raises(TypeError, match="unexpected keyword argument"): + validate_submission(FIXTURE, **{option: tmp_path}) + + +def test_catalog_environment_cannot_replace_validation_approvals(tmp_path, monkeypatch): + monkeypatch.setenv("MLPERF_ENDPOINTS_SEED_SETS", str(tmp_path / "seeds.yaml")) + monkeypatch.setenv( + "MLPERF_ENDPOINTS_APPROVED_DRAFTERS", str(tmp_path / "heads.yaml") + ) + policy = load_policy(bundled_policy_path()) + artifacts = load_artifacts(tmp_path, policy) + assert "seed_sets" in artifacts.catalogs.values + assert not artifacts.catalogs.errors + assert ( + artifacts.catalogs.values["approved_sped_decode_heads"] + == policy.catalogs["approved_sped_decode_heads"] + ) + + +@pytest.mark.parametrize( + "submission", sorted(FIXTURE.parent.iterdir()), ids=lambda path: path.name +) +def test_native_submission_corpus_has_no_internal_evaluation_errors(submission): + report = validate_submission(submission) + assert report.results + assert not any( + "Evaluation could not complete" in finding.message for finding in report.results + ) diff --git a/tests/unit/validation/test_evidence.py b/tests/unit/validation/test_evidence.py index 7a2249e1b..fe89379ee 100644 --- a/tests/unit/validation/test_evidence.py +++ b/tests/unit/validation/test_evidence.py @@ -3,10 +3,16 @@ """Structural evidence parsing and policy/evidence separation.""" import json +from dataclasses import replace import pytest from pydantic import ValidationError +from inference_endpoint.validation import ( + bundled_policy_path, + load_policy, + validate_submission, +) from inference_endpoint.validation.catalogs.seeds import ( SeedSetError, parse_seed_catalog, @@ -44,6 +50,18 @@ def test_native_accuracy_aliases_and_duplicate_datasets(): AccuracyResult.model_validate({"accuracy_scores": [entry, entry]}) +def test_empty_accuracy_is_rejected_by_selected_check(tmp_path): + point = tmp_path / "results" / "system" / "llama3_1-8b" / "r16" + point.mkdir(parents=True) + (point / "accuracy_results.json").write_text("{}") + policy = load_policy(bundled_policy_path()) + rule = next(rule for rule in policy.checks if rule.id == "accuracy-valid") + report = validate_submission(tmp_path, policy=replace(policy, checks=(rule,))) + assert not report.passed + assert report.errors[0].rule == "accuracy-valid" + assert "empty" in report.errors[0].message + + def test_unreadable_encoding_becomes_structured_schema_error(tmp_path): path = tmp_path / "result_summary.json" path.write_bytes(b"\xff") diff --git a/tests/unit/validation/test_policy_contracts.py b/tests/unit/validation/test_policy_contracts.py index e776bf834..395d8ceba 100644 --- a/tests/unit/validation/test_policy_contracts.py +++ b/tests/unit/validation/test_policy_contracts.py @@ -1,12 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +"""Malformed submissions and policy values must not disable validation.""" + +import json import shutil +from pathlib import Path import pytest import yaml from pydantic import ValidationError -from inference_endpoint.validation import bundled_policy_path, load_policy -from inference_endpoint.validation.artifacts import PointArtifacts +from inference_endpoint.validation import ( + bundled_policy_path, + load_policy, + validate_submission, +) +from inference_endpoint.validation.artifacts import PointArtifacts, load_artifacts +from inference_endpoint.validation.checks.power import compute_descriptor from inference_endpoint.validation.evidence.accuracy import AccuracyResult from inference_endpoint.validation.planner import plan_checks from inference_endpoint.validation.power.calculation import PowerCalculator @@ -16,6 +26,33 @@ pytestmark = pytest.mark.unit +@pytest.mark.parametrize( + "score", + [ + True, + False, + {}, + "nonsense", + None, + float("nan"), + float("inf"), + {"exact_match": True}, + {"exact_match": []}, + ], +) +def test_supplied_invalid_accuracy_score_is_schema_failure(tmp_path, score): + point = tmp_path / "results" / "system" / "gpt-oss-120b" / "r16" + point.mkdir(parents=True) + (point / "accuracy_results.json").write_text( + json.dumps({"mlperf_gpt_oss_accuracy": {"num_samples": 4395, "score": score}}) + ) + report = validate_submission(tmp_path) + assert any( + result.rule == "accuracy-valid" and not result.passed + for result in report.results + ) + + def test_accuracy_nested_scores_are_owned(): data = {"aime25": {"score": {"exact_match": 90}, "extras": {"source": "test"}}} parsed = AccuracyResult.model_validate(data) @@ -160,6 +197,30 @@ def test_no_decode_head_keeps_approval_checks_excluded(tmp_path, declaration): assert checks["approved-drafter"].decision is Decision.EXCLUDED +def test_incomplete_canonical_head_does_not_fall_back_to_alias(tmp_path): + policy = load_policy(bundled_policy_path()) + approved = policy.catalogs["approved_sped_decode_heads"][0] + point = tmp_path / "results" / "system" / approved["model"] / "r16" + point.mkdir(parents=True) + config = { + "concurrency": 16, + "runtime_settings": {"runtime": {}}, + "speculative_decoding": {"target_checksum": "declared"}, + "drafter": { + "repository": approved["repository"], + "revision": approved["revision"], + }, + } + (point / "point.yaml").write_text(json.dumps(config)) + report = validate_submission(tmp_path, policy=policy) + assert any( + result.rule == "approved-drafter" + and not result.passed + and "approved identity" in result.message + for result in report.results + ) + + def test_valid_override_retains_false_and_numeric_values(policy_dir): path = policy_dir / "catalog.yaml" document = yaml.safe_load(path.read_text()) @@ -209,3 +270,81 @@ def test_malformed_nested_and_operation_contracts_are_rejected( path.write_text(yaml.safe_dump(document, sort_keys=False)) with pytest.raises(ValueError, match=rule): load_policy(policy_dir) + + +@pytest.mark.parametrize( + ("description", "descriptor_cooling", "accepted"), + [ + ("air", "air", True), + ("water-cooled", "liquid", True), + ("mixed", "mixed", True), + ("air and liquid", "mixed", True), + ("air and liquid", "air", False), + ("liquid", "mixed", False), + ], +) +def test_power_cooling_must_match_description( + tmp_path, description, descriptor_cooling, accepted +): + policy = load_policy(bundled_policy_path()) + source = ( + Path(__file__).resolve().parents[3] + / "tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/system_desc.json" + ) + system = tmp_path / "results" / "system" + point = system / "gpt-oss-120b" / "r16" + point.mkdir(parents=True) + system_json = json.loads(source.read_text()) + system_json["cooling"] = description + for node in system_json["node_types"]: + node.pop("cooling", None) + (point / "system_desc.json").write_text(json.dumps(system_json)) + (point / "point.yaml").write_text( + json.dumps( + { + "concurrency": 16, + "division": "Standardized", + "runtime_settings": {"runtime": {}}, + } + ) + ) + (system / "system_power.json").write_text( + json.dumps( + { + "system_desc_id": "system", + "cooling": descriptor_cooling, + "node_sets": [ + { + "node_set_id": 1, + "system_node_ensemble_id": 1, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "combined_cpu_accelerator": { + "value_w": 9800, + "source_type": "vendor_spec", + "source": "https://example.com/power", + }, + "scale_up_network": {"method": "none"}, + }, + } + ], + "scale_out": {"present": False}, + } + ) + ) + artifacts = load_artifacts(tmp_path, policy) + subject = next( + subject for subject in artifacts.index.subjects if subject.id == str(system) + ) + check = next( + check + for check in plan_checks(policy, [subject]).ready + if check.rule.id == "power-descriptor" + ) + compute_descriptor(check, artifacts) + entry = artifacts.derived.power[str(system)] + assert (entry["computation"].provisioned_power_kw is not None) is accepted + assert ("Power cooling disagrees with system description" in entry["problems"]) is ( + not accepted + ) diff --git a/tests/unit/validation/test_review_regressions.py b/tests/unit/validation/test_review_regressions.py new file mode 100644 index 000000000..c38c58dab --- /dev/null +++ b/tests/unit/validation/test_review_regressions.py @@ -0,0 +1,193 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Catalog defaults and member-level policy exception contracts.""" + +import shutil + +import pytest +import yaml + +from inference_endpoint.validation import ( + Context, + Decision, + bundled_policy_path, + load_policy, + plan_checks, +) + +pytestmark = pytest.mark.unit + + +def with_override(tmp_path, rule, conditions): + directory = tmp_path / "2026-10-C1" + shutil.copytree(bundled_policy_path(), directory) + path = directory / "catalog.yaml" + data = yaml.safe_load(path.read_text()) + data["enrollment"]["overrides"] = [ + { + "id": "member-exception", + "rule": rule, + "reason": "Exception for matching points", + "applies_to": conditions, + "enabled": False, + } + ] + path.write_text(yaml.safe_dump(data, sort_keys=False)) + return load_policy(directory) + + +def test_loaded_catalog_includes_parsed_dataset_defaults(): + dataset = load_policy(bundled_policy_path()).catalogs["datasets"]["cnn_dailymail"] + assert dataset["is_legacy"] is False + assert dataset["sample_unit"] == "sample" + + +def test_curve_exception_uses_selected_member_facts(tmp_path): + policy = with_override(tmp_path, "offline-ordering", {"offline": ["dedicated"]}) + curve = Context( + id="kimi", + scope="curve", + model_id="kimi-k3", + offline="none", + members=(Context(id="r4096", scope="point", offline="dedicated"),), + ) + check = next( + check + for check in plan_checks(policy, [curve]).checks + if check.rule.id == "offline-ordering" + ) + assert check.decision is Decision.EXCLUDED + assert check.override == "member-exception" + assert check.selected_members == ("r4096",) + + +def test_curve_exception_partitions_members_without_hiding_other_points(tmp_path): + policy = with_override( + tmp_path, "benchmark-type-consistency", {"load_pattern": ["agentic_inference"]} + ) + curve = Context( + id="mixed", + scope="curve", + model_id="kimi-k3", + members=( + Context( + id="a", scope="point", offline="none", load_pattern="agentic_inference" + ), + Context(id="b", scope="point", offline="none", load_pattern="concurrency"), + ), + ) + checks = [ + check + for check in plan_checks(policy, [curve]).checks + if check.rule.id == "benchmark-type-consistency" + ] + assert len(checks) == 2 + assert {(check.decision, check.selected_members) for check in checks} == { + (Decision.EXCLUDED, ("a",)), + (Decision.READY, ("b",)), + } + + +def test_unknown_point_facts_are_not_filled_from_curve_declarations(): + policy = load_policy(bundled_policy_path()) + curve = Context( + id="curve", + scope="curve", + model_id="kimi-k3", + offline="dedicated", + members=(Context(id="unknown", scope="point"),), + ) + check = next( + check + for check in plan_checks(policy, [curve]).checks + if check.rule.id == "benchmark-type-consistency" + ) + assert check.decision is Decision.BLOCKED + + +def test_unknown_member_override_blocks_only_its_partition(tmp_path): + policy = with_override( + tmp_path, "benchmark-type-consistency", {"load_pattern": ["agentic_inference"]} + ) + curve = Context( + id="mixed", + scope="curve", + model_id="kimi-k3", + load_pattern="concurrency", + members=( + Context( + id="known", scope="point", offline="none", load_pattern="concurrency" + ), + Context(id="unknown", scope="point", offline="none"), + ), + ) + checks = [ + check + for check in plan_checks(policy, [curve]).checks + if check.rule.id == "benchmark-type-consistency" + ] + assert {(check.decision, check.selected_members) for check in checks} == { + (Decision.READY, ("known",)), + (Decision.BLOCKED, ("unknown",)), + } + + +def test_enabled_curve_overrides_keep_distinct_immutable_requirements(tmp_path): + with_override(tmp_path, "offline-ordering", {"offline": ["dedicated"]}) + path = tmp_path / "2026-10-C1/catalog.yaml" + data = yaml.safe_load(path.read_text()) + data["enrollment"]["overrides"] = [ + { + "id": pattern, + "rule": "offline-ordering", + "reason": "Pattern-specific ordering tolerance", + "applies_to": {"load_pattern": [pattern]}, + "requirements": {"throughput_minimum_multiplier": value}, + } + for pattern, value in [("max_throughput", 0.9), ("agentic_inference", 0.8)] + ] + path.write_text(yaml.safe_dump(data, sort_keys=False)) + policy = load_policy(path.parent) + curve = Context( + id="curve", + scope="curve", + model_id="kimi-k3", + members=tuple( + Context( + id=pattern, scope="point", offline="dedicated", load_pattern=pattern + ) + for pattern in ["max_throughput", "agentic_inference"] + ), + ) + checks = [ + check + for check in plan_checks(policy, [curve]).checks + if check.rule.id == "offline-ordering" + ] + assert { + (check.override, check.rule.requirements["throughput_minimum_multiplier"]) + for check in checks + } == {("max_throughput", 0.9), ("agentic_inference", 0.8)} + assert all(check.decision is Decision.READY for check in checks) + with pytest.raises(TypeError): + checks[0].rule.requirements["throughput_minimum_multiplier"] = 1 + original = next(rule for rule in policy.checks if rule.id == "offline-ordering") + assert original.requirements["throughput_minimum_multiplier"] == 0.98 + + +def test_overlapping_overrides_are_detected_on_each_curve_member(tmp_path): + with_override(tmp_path, "offline-ordering", {"offline": ["dedicated"]}) + path = tmp_path / "2026-10-C1/catalog.yaml" + data = yaml.safe_load(path.read_text()) + data["enrollment"]["overrides"].append( + {**data["enrollment"]["overrides"][0], "id": "second"} + ) + path.write_text(yaml.safe_dump(data, sort_keys=False)) + curve = Context( + id="curve", + scope="curve", + model_id="kimi-k3", + members=(Context(id="point", scope="point", offline="dedicated"),), + ) + with pytest.raises(ValueError, match="Overlapping overrides.*point"): + plan_checks(load_policy(path.parent), [curve]) diff --git a/tests/unit/validation/test_special_evaluators.py b/tests/unit/validation/test_special_evaluators.py new file mode 100644 index 000000000..b9417b09c --- /dev/null +++ b/tests/unit/validation/test_special_evaluators.py @@ -0,0 +1,477 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Behavioral policy mutations for special evaluators.""" + +import json +from dataclasses import replace +from pathlib import Path + +import pytest + +from inference_endpoint.validation import ( + Conditions, + Context, + bundled_policy_path, + checks, # noqa: F401 - registers built-in evaluators + load_policy, + validate_submission, +) +from inference_endpoint.validation.artifacts import ( + ArtifactIndex, + Artifacts, + PointArtifacts, + load_artifacts, +) +from inference_endpoint.validation.catalogs.seeds import SeedSet +from inference_endpoint.validation.checks.power import prepare +from inference_endpoint.validation.evaluator_base import Evaluator +from inference_endpoint.validation.evidence.accuracy import AccuracyResult +from inference_endpoint.validation.evidence.loaders import ParsedArtifact +from inference_endpoint.validation.models import Override, thaw +from inference_endpoint.validation.planner import PlannedCheck +from inference_endpoint.validation.power.calculation import PowerCalculator +from inference_endpoint.validation.power.models import SystemPower +from inference_endpoint.validation.types import Decision, EvidenceKey, Scope + +pytestmark = pytest.mark.unit + + +def update_evidence(point, field, **updates): + artifact = getattr(point.evidence, field) + data = artifact.value.model_dump( + mode="json", exclude_unset=True, exclude_computed_fields=True + ) + data.update(updates) + parsed = ParsedArtifact.from_json(data, type(artifact.value), point.path, field) + assert not parsed.errors + point.evidence = replace(point.evidence, **{field: parsed}) + + +def update_accuracy(point, data): + parsed = ParsedArtifact.from_json( + data, AccuracyResult, point.path, "accuracy-valid" + ) + assert not parsed.errors + point.evidence = replace(point.evidence, accuracy=parsed) + + +def evaluate(check, artifacts): + return Evaluator.lookup(check.rule.kind)()(check, artifacts) + + +@pytest.fixture +def artifacts(): + policy = load_policy(bundled_policy_path()) + point = PointArtifacts.from_json( + Path("/submission/results/system/gpt-oss-120b/r16"), + { + "model_name": "gpt-oss-120b", + "concurrency": 16, + "offline": "elected", + "runtime_settings": {"load_pattern": "concurrency", "runtime": {}}, + }, + {"n_samples_completed": 1, "duration_ns": 1e9}, + {}, + ) + point.context = Context( + id=str(point.path), scope=Scope.POINT, model_id="gpt-oss-120b" + ) + return Artifacts( + Path("/submission"), + policy, + index=ArtifactIndex( + points={str(point.path): point}, + curves={str(point.curve): [str(point.path)]}, + ), + ) + + +def check_for(artifacts, rule_id, **parameters): + rule = next(rule for rule in artifacts.policy.checks if rule.id == rule_id) + if parameters: + rule = replace(rule, requirements={**rule.requirements, **parameters}) + point = next(iter(artifacts.index.points.values())) + subject = ( + point.context + if rule.scope is Scope.POINT + else Context( + id=str(point.curve), scope=rule.scope, model_id=point.context.model_id + ) + ) + return PlannedCheck(subject, rule, Decision.READY, "Applicable") + + +def point(artifacts): + return next(iter(artifacts.index.points.values())) + + +def test_accuracy_catalog_threshold_changes_acceptance(artifacts): + p = point(artifacts) + update_accuracy( + p, + { + "mlperf_gpt_oss_accuracy": { + "num_samples": 4395, + "score": {"exact_match": 83}, + } + }, + ) + check = check_for(artifacts, "accuracy-gate") + assert all(result.passed for result in evaluate(check, artifacts)) + catalogs = thaw(artifacts.policy.catalogs) + catalogs["models"]["gpt-oss-120b"]["accuracy"]["metrics"]["exact_match"][ + "reference" + ] = 90 + artifacts.policy = replace(artifacts.policy, catalogs=catalogs) + assert any(not result.passed for result in evaluate(check, artifacts)) + + +def test_unrelated_accuracy_dataset_cannot_satisfy_required_gate(artifacts): + update_accuracy( + point(artifacts), + {"other": {"num_samples": 999999, "score": {"exact_match": 100}}}, + ) + results = evaluate(check_for(artifacts, "accuracy-gate"), artifacts) + assert len(results) == 1 + assert not results[0].passed + assert "Required accuracy datasets missing" in results[0].message + + +def test_accuracy_count_uses_repeat_policy(artifacts): + update_accuracy( + point(artifacts), {"mlperf_gpt_oss_accuracy": {"num_samples": 3000, "score": 1}} + ) + assert any( + not r.passed + for r in evaluate(check_for(artifacts, "accuracy-sample-count"), artifacts) + ) + assert all( + r.passed + for r in evaluate( + check_for(artifacts, "accuracy-sample-count", missing_repeats=2), artifacts + ) + ) + + +def test_steady_superpass_catalog_controls_official_window(artifacts): + update_evidence( + point(artifacts), + "config", + steady_state={ + "status": "windowable", + "verdict": "steady_state", + "state": {"tpot_p90": "plateau"}, + "window": {"super_pass_start": 1, "super_pass_end": 4, "n_super_passes": 4}, + }, + ) + check = check_for(artifacts, "steady-state-consistency") + assert all(r.passed for r in evaluate(check, artifacts)) + catalogs = thaw(artifacts.policy.catalogs) + catalogs["steady_state"]["minimum_super_passes"] = 5 + artifacts.policy = replace(artifacts.policy, catalogs=catalogs) + assert any(not r.passed for r in evaluate(check, artifacts)) + + +def test_seed_adoption_window_parameter_and_unpublished_cohort(artifacts): + p = point(artifacts) + update_evidence(p, "config", seed_set="A", target_cohort="2026-11-C0") + artifacts.catalogs.values["seed_sets"] = { + "A": SeedSet( + "A", 1, 2, 3, ("2026-10-C0", "2026-10-C1", "2026-11-C0", "2026-11-C1") + ) + } + assert all( + r.passed for r in evaluate(check_for(artifacts, "seed-set-adoption"), artifacts) + ) + assert any( + not r.passed + for r in evaluate( + check_for(artifacts, "seed-set-adoption", adoption_window_cohorts=2), + artifacts, + ) + ) + artifacts.catalogs.values["seed_sets"]["A"] = SeedSet("A", 1, 2, 3) + results = evaluate(check_for(artifacts, "seed-set-adoption"), artifacts) + assert results[0].key == "blocked" + assert not results[0].passed + + +def test_accuracy_coverage_overlapping_band_and_elected_offline(artifacts): + p = point(artifacts) + update_accuracy(p, {"mlperf_gpt_oss_accuracy": {"score": 1}}) + artifacts.derived.regions[str(p.curve)] = { + "low_concurrency": (2, 32), + "med_concurrency": (33, 64), + "high_concurrency": (65, 128), + } + catalogs = thaw(artifacts.policy.catalogs) + catalogs["mandatory_accuracy_bands"] = ["ultra_low_concurrency", "low_concurrency"] + artifacts.policy = replace(artifacts.policy, catalogs=catalogs) + assert all( + r.passed for r in evaluate(check_for(artifacts, "accuracy-coverage"), artifacts) + ) + + +def test_power_catalog_overhead_and_component_defaults(artifacts): + descriptor = SystemPower.model_validate( + { + "system_desc_id": "system", + "cooling": "air", + "node_sets": [ + { + "node_set_id": 1, + "system_node_ensemble_id": 1, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "cpu": {"model": "AMD EPYC 64-core", "count_per_node": 1}, + "accelerator": {"model": "NVIDIA B200", "count_per_node": 1}, + "scale_up_network": {"method": "none"}, + }, + } + ], + "scale_out": {"present": False}, + } + ) + catalog = thaw(artifacts.policy.catalogs["power"]) + default = PowerCalculator(descriptor, catalog).compute() + assert default.provisioned_power_kw == 2.02 + catalog["defaults"]["accelerators"]["b200"] = 2000 + catalog["cooling_overhead"]["air"] = 0.1 + changed = PowerCalculator(descriptor, catalog).compute() + assert changed.provisioned_power_kw == 2.58 + assert changed.estimated + + +def test_unknown_special_operation_is_blocked(artifacts): + results = evaluate( + check_for(artifacts, "warmup-salt", operation="unsupported"), artifacts + ) + assert results[0].key == "blocked" + assert not results[0].passed + + +def test_drafter_identity_requires_repository_and_revision(artifacts): + p = point(artifacts) + p.context = p.context.model_copy(update={"model_id": "kimi-k3"}) + update_evidence( + p, "config", speculative_decoding={"weight_checksum": "same-checksum"} + ) + results = evaluate(check_for(artifacts, "approved-drafter"), artifacts) + assert any(not result.passed for result in results) + artifacts.catalogs.values["approved_sped_decode_heads"] = artifacts.policy.catalogs[ + "approved_sped_decode_heads" + ] + approved = artifacts.policy.catalogs["approved_sped_decode_heads"][0] + update_evidence( + p, + "config", + speculative_decoding={ + "repository": approved["repository"], + "revision": approved["revision"], + }, + ) + assert all( + result.passed + for result in evaluate(check_for(artifacts, "approved-drafter"), artifacts) + ) + + +def test_invalid_reported_superpass_count_fails(artifacts): + update_evidence( + point(artifacts), + "config", + steady_state={ + "status": "windowable", + "state": {"tpot_p90": "plateau"}, + "window": {"super_pass_start": 1, "super_pass_end": 4, "n_super_passes": 5}, + }, + ) + assert any( + not result.passed + for result in evaluate( + check_for(artifacts, "steady-state-consistency"), artifacts + ) + ) + assert all( + result.passed + for result in evaluate( + check_for( + artifacts, + "steady-state-consistency", + check_reported_super_pass_count=False, + ), + artifacts, + ) + ) + + +@pytest.fixture +def two_power_systems(tmp_path): + policy = load_policy(bundled_policy_path()) + payload = { + "system_desc_id": "system", + "cooling": "air", + "node_sets": [ + { + "node_set_id": 1, + "system_node_ensemble_id": 1, + "nodes_provisioned": 1, + "power_method": "component_sum", + "components": { + "combined_cpu_accelerator": { + "value_w": 9800, + "source_type": "vendor_spec", + "source": "https://example.com/power", + }, + "scale_up_network": {"method": "none"}, + }, + } + ], + "scale_out": {"present": False}, + } + for name, division in [("system_a", "Standardized"), ("system_b", "Serviced")]: + system = tmp_path / "results" / name + p = system / "gpt-oss-120b" / "r16" + p.mkdir(parents=True) + (system / "system_power.json").write_text(json.dumps(payload)) + (p / "point.yaml").write_text( + f"concurrency: 16\nmodel_name: gpt-oss-120b\ndivision: {division}\nruntime_settings:\n load_pattern: concurrency\n runtime: {{}}\n" + ) + return load_artifacts(tmp_path, policy) + + +def test_power_descriptor_override_changes_only_selected_system(two_power_systems): + artifacts = two_power_systems + catalogs = thaw(artifacts.policy.catalogs) + constants = catalogs["power"] + constants["cooling_overhead"]["air"] = 1.0 + artifacts.policy = replace( + artifacts.policy, + overrides=( + Override( + "air-overhead", + "power-descriptor", + "System-specific overhead", + Conditions(division=["Standardized"]), + True, + {"constants": constants}, + ), + ), + ) + prepare(artifacts) + values = { + Path(id).name: entry["computation"].provisioned_power_kw + for id, entry in artifacts.derived.power.items() + } + assert values == {"system_a": 19.6, "system_b": 14.7} + + +def test_disabled_point_power_never_provides_denominator(two_power_systems): + artifacts = two_power_systems + artifacts.policy = replace( + artifacts.policy, + overrides=( + Override( + "no-denominator", + "point-power", + "Disabled", + Conditions(division=["Standardized"]), + False, + {}, + ), + ), + ) + prepare(artifacts) + for p in artifacts.index.points.values(): + if p.context.division.value == "Standardized": + assert p.derived.power_kw is None + assert EvidenceKey.POINT_POWER not in p.context.available + else: + assert p.derived.power_kw == 14.7 + assert EvidenceKey.POINT_POWER in p.context.available + + +def test_point_power_rounding_override_applies_during_preparation(two_power_systems): + artifacts = two_power_systems + artifacts.policy = replace( + artifacts.policy, + overrides=( + Override( + "whole-kw", + "point-power", + "Round selected point", + Conditions(division=["Standardized"]), + True, + {"round_kw_decimals": 0}, + ), + ), + ) + prepare(artifacts) + values = { + p.path.parent.parent.name: p.derived.power_kw + for p in artifacts.index.points.values() + } + assert values == {"system_a": 15.0, "system_b": 14.7} + + +@pytest.mark.parametrize( + "enabled,parameters", [(False, {}), (True, {"validate_computed_values": False})] +) +def test_unevaluated_descriptor_provides_no_power_facts( + two_power_systems, enabled, parameters +): + artifacts = two_power_systems + artifacts.policy = replace( + artifacts.policy, + overrides=( + Override( + "no-system-power", + "power-descriptor", + "Unavailable power", + Conditions(division=["Standardized"]), + enabled, + parameters, + ), + ), + ) + prepare(artifacts) + selected = next( + p + for p in artifacts.index.points.values() + if p.context.division.value == "Standardized" + ) + assert selected.derived.power_kw is None + assert EvidenceKey.POWER_COMPUTATION not in selected.context.available + assert EvidenceKey.POINT_POWER not in selected.context.available + other = next( + p + for p in artifacts.index.points.values() + if p.context.division.value == "Serviced" + ) + assert other.derived.power_kw == 14.7 + + +def test_invalid_point_power_policy_returns_blocked_report(two_power_systems): + artifacts = two_power_systems + effective_checks = tuple( + replace( + rule, requirements={**rule.requirements, "round_kw_decimals": "invalid"} + ) + if rule.id == "point-power" + else rule + for rule in artifacts.policy.checks + ) + policy = replace(artifacts.policy, checks=effective_checks) + report = validate_submission(artifacts.root, policy=policy) + assert any( + result.rule == "point-power" and result.key == "blocked" + for result in report.errors + ) + artifacts.policy = policy + prepare(artifacts) + assert all( + point.derived.power_kw is None + and EvidenceKey.POINT_POWER not in point.context.available + for point in artifacts.index.points.values() + ) From 2ef463fe1ac61cba337427f16111bfc290704194 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 01:35:21 -0700 Subject: [PATCH 07/18] feat(validation): expose submission-check command and documentation --- AGENTS.md | 11 ++ docs/validation/README.md | 129 ++++++++++++++ docs/validation/follow-ups.md | 6 + docs/validation/implementation.md | 163 ++++++++++++++++++ pyproject.toml | 1 + src/inference_endpoint/validation/__main__.py | 75 ++++++++ tests/unit/validation/test_cli.py | 52 ++++++ 7 files changed, 437 insertions(+) create mode 100644 docs/validation/README.md create mode 100644 docs/validation/follow-ups.md create mode 100644 docs/validation/implementation.md create mode 100644 src/inference_endpoint/validation/__main__.py create mode 100644 tests/unit/validation/test_cli.py diff --git a/AGENTS.md b/AGENTS.md index 4bc422592..e40741d6a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -100,6 +100,7 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint | **VideoGen** | `src/inference_endpoint/videogen/` | Adapter for video-generation endpoints (e.g. trtllm-serve `POST /v1/videos/generations`, used by MLPerf WAN2.2-T2V-A14B). Defaults to `response_format=video_path` (server saves video to shared storage and returns path) to avoid large byte payloads. Accuracy mode also runs on `video_path`: the adapter mirrors the path into `response_output` so the event log carries it to `VBenchScorer` (see `evaluation/scoring.py`), which scores videos via VBench from a sibling `uv` subproject at `examples/09_Wan22_VideoGen_Example/accuracy/` (vbench's `transformers==4.33.2` + `numpy<2` pins are incompatible with the parent env, so it runs out-of-process via `uv run --project`). Dataset is ingested via the generic JSONL loader. | | **SWE-bench** | `src/inference_endpoint/dataset_manager/predefined/swe_bench/`, `src/inference_endpoint/evaluation/swe_bench_scorer.py`, `src/inference_endpoint/evaluation/swebench_service/` | `SWEBench` predefined dataset (HuggingFace `princeton-nlp/SWE-bench_Verified` or `_Lite`; `ACCURACY_ONLY=True`). `SWEBenchScorer` sets `SKIP_ENDPOINT_PHASE=True` and bypasses the built-in accuracy phase entirely: it delegates agent execution and grading to the configured SWE-bench service via `accuracy_config.extras.swebench_service_url`. The service is an isolated `uv` subproject; its host owns Docker/runtime execution, artifacts, and credentials, while the benchmark client remains the report-producing entrypoint. | | **Compliance (submission checker)** | `src/inference_endpoint/compliance/checker.py`, `scripts/check_compliance.py` | Validates a completed run's report directory against a registered ruleset. `check_submission(report_dir, ruleset, model)` reads the resolved `config.yaml` plus scorer output (`accuracy/accuracy_results.json` for accuracy, `scores.json` for the agentic perf run) and runs config-lock (deterministic + single-stream), the accuracy gate (`score >= factor x reference`, factor 0.97 for Edge-Agentic), and run validity (0 dropped turns). Server-side launch flags (`--reasoning off`, `--ctx-size`) aren't in client artifacts, so they're surfaced as manual attestations. CLI: `scripts/check_compliance.py REPORT_DIR` (exit 0 = pass). | +| **Submission validation** | `src/inference_endpoint/validation/` | Artifact validation, five-file cohort policies, typed evidence schemas with supplied-field metadata, grouped artifact indexes and derived state, check planning and policy-driven power calculations. See `docs/validation/implementation.md`. | | **Compliance (audit tests)** | `src/inference_endpoint/compliance/`, `commands/audit.py` | MLPerf compliance audits. `AuditTest` protocol + `AuditRunSpec`/`AuditRunArtifacts` + registry (`compliance/__init__.py`); `OutputCachingAudit` (`compliance/audit_test/output_caching_test.py`, which also owns the QPS-specific `AuditRunStats`) implements MLPerf **TEST04** output-caching detection — reference phase (distinct samples) vs. fixed-sample audit phase, comparing QPS against `threshold`. `commands/audit.py:run_audit` runs phases via `AuditTest.plan_runs`/`validate`, writing `audit_result.json`/`verify_.txt` atomically via `compliance/result.py`. Enabled by the `audit:` YAML block; `cli._run` runs it after the main benchmark (upstream MLPerf order: perf run, then TEST04), or standalone with `audit.only: true`. Perf-only by default (a phase may opt into accuracy via `AuditRunSpec.test_mode`, but this is unused today). | ### Hot-Path Architecture @@ -287,6 +288,16 @@ src/inference_endpoint/ ├── compliance/ # Submission compliance checks (config-lock, accuracy gate, run validity) │ ├── __init__.py │ └── checker.py # check_submission() + Check/ComplianceReport (Edge-Agentic ruleset) +├── validation/ # Cohort policies, evidence schemas, planner, evaluators and power arithmetic +│ ├── policies/ # Versioned cohort bundles (catalog and scoped checks) +│ ├── schemas/ # Cohort/revision parsers and typed requirement contracts +│ ├── operations.py # Shared policy and evaluator operation enums +│ ├── planner.py # Inclusion, exclusions, overrides and prerequisites +│ ├── checks/ # Callable check implementations +│ ├── evaluator_base.py # Evaluator base class and subclass registry +│ ├── evidence/ # Parsed artifact schemas and field-presence metadata +│ ├── artifacts.py # Artifact construction, indexes and classification +│ └── api.py # Policy inspection and submission validation ├── plugins/ # Plugin system ├── profiling/ # line_profiler integration, pytest plugin ├── testing/ diff --git a/docs/validation/README.md b/docs/validation/README.md new file mode 100644 index 000000000..cdf3dce00 --- /dev/null +++ b/docs/validation/README.md @@ -0,0 +1,129 @@ +# Cohort policy coverage + +The 2026-10-C1 revision 1 bundle contains 89 definitions across five files in +`src/inference_endpoint/validation/policies/2026-10-C1/`. +Catalogs, enrollment, exceptions and catalog diagnostics live in catalog.yaml. +All files declare the same cohort and revision; parser intervals are inclusive. + +## Current interfaces + +The artifact validation API loads the five-file cohort policy, classifies the +submission, plans applicable checks, and executes registered evaluators. Its +regression suite covers artifact parsing, disclosures, seeds, accuracy, metric +consistency, coverage, and provisioned/per-point power. + +Check selection uses typed classifications and prerequisites. Ready means selected +with available prerequisites; evaluators determine compliance. Requirement field +names and references are validated by the versioned policy parser. + +## Conditions and evidence + +Each check declares inline inclusion/exclusion predicates and dependencies. +Different fields are ANDed; enum lists are alternatives. Exclusion wins. +Unknown classification blocks selection. Empty fact sets represent known absence. +Curve member exceptions produce distinct plans with selected member IDs. +Overlapping exceptions on one member fail explicitly. Point-specific unknown +facts remain unknown. Independent checks stay planned when another artifact fails. +Datasets normalize is_legacy to false and sample_unit to sample when omitted. +Seeds come from the bundled published cohort catalog. + +## Unresolved policy inputs + +The approved_client_revisions list is empty. Published full client SHAs must be +supplied before client revision approval can pass. +Speculative-head approval publication cohorts are unspecified. +The policy catalog is proposed, not an official MLCommons publication. + +## Definitions + +| Rule ID | Scope | Evaluator kind | Policy file | +| ---------------------------------- | -------------- | ------------------- | ------------------------ | +| `accuracy-coverage` | curve | `accuracy_coverage` | `curve_checks.yaml` | +| `accuracy-gate` | curve | `accuracy_gate` | `curve_checks.yaml` | +| `accuracy-present` | submission | `accuracy_presence` | `submission_checks.yaml` | +| `accuracy-sample-count` | curve | `accuracy_gate` | `curve_checks.yaml` | +| `accuracy-valid` | point | `artifact_schema` | `point_checks.yaml` | +| `agentic-accuracy` | curve | `accuracy_gate` | `curve_checks.yaml` | +| `agentic-accuracy-inline` | curve | `accuracy_gate` | `curve_checks.yaml` | +| `agentic-accuracy-swebench` | curve | `accuracy_gate` | `curve_checks.yaml` | +| `agentic-metric-consistency` | point | `derived_metric` | `point_checks.yaml` | +| `agentic-osl-range` | curve | `accuracy_gate` | `curve_checks.yaml` | +| `agentic-trajectory-count` | point | `dataset_count` | `point_checks.yaml` | +| `approved-checkpoint` | point | `artifact_binding` | `point_checks.yaml` | +| `approved-drafter` | point | `drafter_binding` | `point_checks.yaml` | +| `benchmark-model-dir` | system | `presence` | `system_checks.yaml` | +| `benchmark-type-consistency` | curve | `consistency` | `curve_checks.yaml` | +| `concurrency-in-range` | point | `region_placement` | `point_checks.yaml` | +| `config-consistency-dataset` | curve | `consistency` | `curve_checks.yaml` | +| `config-consistency-model` | curve | `consistency` | `curve_checks.yaml` | +| `drafter-approval-lead-time` | point | `drafter_binding` | `point_checks.yaml` | +| `drafter-list-registry` | validator | `catalog_integrity` | `catalog.yaml` | +| `endpoints-client-sha` | point | `artifact_binding` | `point_checks.yaml` | +| `high-concurrency-coverage` | curve | `coverage` | `curve_checks.yaml` | +| `load-pattern` | point | `issuance` | `point_checks.yaml` | +| `low-concurrency-coverage` | curve | `coverage` | `curve_checks.yaml` | +| `max-concurrency-declared` | curve | `report` | `curve_checks.yaml` | +| `maximal-engagement` | point | `power` | `point_checks.yaml` | +| `measurement-points-present` | curve | `presence` | `curve_checks.yaml` | +| `med-concurrency-coverage` | curve | `coverage` | `curve_checks.yaml` | +| `metric-consistency-accounting` | point | `comparison` | `point_checks.yaml` | +| `metric-consistency-duration` | point | `comparison` | `point_checks.yaml` | +| `metric-consistency-output-tokens` | point | `comparison` | `point_checks.yaml` | +| `metric-consistency-system-tps` | point | `derived_metric` | `point_checks.yaml` | +| `metric-consistency-tpot-p90` | point | `numeric_validity` | `point_checks.yaml` | +| `metric-consistency-tps-per-kw` | point | `derived_metric` | `point_checks.yaml` | +| `metric-consistency-tps-per-user` | point | `derived_metric` | `point_checks.yaml` | +| `min-query-count` | point | `dataset_minimum` | `point_checks.yaml` | +| `model-name-consistency` | curve | `consistency` | `curve_checks.yaml` | +| `model-name-valid` | curve | `membership` | `curve_checks.yaml` | +| `nodes-used` | point | `power` | `point_checks.yaml` | +| `offline-declared` | point | `membership` | `point_checks.yaml` | +| `offline-ordering` | curve | `offline` | `curve_checks.yaml` | +| `offline-point-present` | curve | `offline` | `curve_checks.yaml` | +| `path-exists` | submission | `presence` | `submission_checks.yaml` | +| `point-cap` | curve | `collection_size` | `curve_checks.yaml` | +| `point-config-valid` | point | `artifact_schema` | `point_checks.yaml` | +| `point-count` | curve | `collection_size` | `curve_checks.yaml` | +| `point-dirname-concurrency` | point | `consistency` | `point_checks.yaml` | +| `point-dirs` | curve | `presence` | `curve_checks.yaml` | +| `point-disclosure-complete` | point | `disclosure` | `point_checks.yaml` | +| `point-duration` | point | `duration` | `point_checks.yaml` | +| `point-power` | point | `power` | `point_checks.yaml` | +| `point-rules-skipped` | point | `report` | `point_checks.yaml` | +| `power-descriptor` | system | `power` | `system_checks.yaml` | +| `power-estimated` | system | `power` | `system_checks.yaml` | +| `region-basis` | curve | `region_basis` | `curve_checks.yaml` | +| `region-computation` | curve | `region_boundaries` | `curve_checks.yaml` | +| `region-declared` | point | `membership` | `point_checks.yaml` | +| `region-placement` | point | `region_placement` | `point_checks.yaml` | +| `required-dir` | submission | `presence` | `submission_checks.yaml` | +| `result-file-valid` | point | `artifact_schema` | `point_checks.yaml` | +| `result-summary-present` | point | `presence` | `point_checks.yaml` | +| `seed-config-legacy` | point | `seed_binding` | `point_checks.yaml` | +| `seed-runtime-match` | point | `seed_binding` | `point_checks.yaml` | +| `seed-set-adoption` | point | `seed_binding` | `point_checks.yaml` | +| `seed-set-consistency` | curve | `consistency` | `curve_checks.yaml` | +| `seed-set-membership` | point | `seed_binding` | `point_checks.yaml` | +| `seed-set-registry` | validator | `catalog_integrity` | `catalog.yaml` | +| `shared-path-resolution` | point | `path_resolution` | `point_checks.yaml` | +| `src-dir` | submission | `presence` | `submission_checks.yaml` | +| `src-readme` | implementation | `presence` | `submission_checks.yaml` | +| `steady-state-basis` | point | `steady_state` | `point_checks.yaml` | +| `steady-state-consistency` | point | `steady_state` | `point_checks.yaml` | +| `steady-state-valid` | point | `steady_state` | `point_checks.yaml` | +| `streaming-config` | point | `report` | `point_checks.yaml` | +| `submission-flags` | point | `field_constraints` | `point_checks.yaml` | +| `swebench-instance-count` | point | `dataset_count` | `point_checks.yaml` | +| `swebench-template` | point | `field_constraints` | `point_checks.yaml` | +| `system-description-across-curves` | system | `consistency` | `system_checks.yaml` | +| `system-description-consistency` | curve | `consistency` | `curve_checks.yaml` | +| `system-description-present` | point | `presence` | `point_checks.yaml` | +| `system-description-valid` | point | `artifact_schema` | `point_checks.yaml` | +| `system-name-consistency` | submission | `consistency` | `submission_checks.yaml` | +| `system-results-dir` | submission | `presence` | `submission_checks.yaml` | +| `target-cohort` | point | `cohort_identifier` | `point_checks.yaml` | +| `tps-utilization` | curve | `derived_metric` | `curve_checks.yaml` | +| `ultra-low-concurrency-coverage` | curve | `coverage` | `curve_checks.yaml` | +| `warmup-logs-retained` | point | `warmup` | `point_checks.yaml` | +| `warmup-present` | point | `presence` | `point_checks.yaml` | +| `warmup-salt` | point | `warmup` | `point_checks.yaml` | diff --git a/docs/validation/follow-ups.md b/docs/validation/follow-ups.md new file mode 100644 index 000000000..5953b4636 --- /dev/null +++ b/docs/validation/follow-ups.md @@ -0,0 +1,6 @@ +# Validation follow-ups + +- **Lower priority — standalone CLI reporting:** print a human-readable summary + and actionable validation findings, and save the structured JSON report as an + artifact. Keep JSON available for automated consumers. Decide the output-file + option and default destination when implementing this change. diff --git a/docs/validation/implementation.md b/docs/validation/implementation.md new file mode 100644 index 000000000..c41f491e9 --- /dev/null +++ b/docs/validation/implementation.md @@ -0,0 +1,163 @@ +# Submission validation + +`inference_endpoint.validation` validates submission artifacts against a versioned +cohort policy. The command-line entry point is `inference-endpoint-validation`. + +```mermaid +flowchart LR + Y[Five cohort YAML files] --> L[Loader and revision parser] + L --> P[Immutable policy] + A[Submission artifacts] --> C[Typed artifact parsing and classification] + S[Published seed catalog] --> C + P --> T[Check planner] + C --> T + T --> E[Registered callable evaluators] + C --> E + E --> R[Structured findings and report] + T --> R +``` + +## Execution + +```python +from pathlib import Path + +from inference_endpoint.validation import validate_submission + +report = validate_submission(Path("submissions/acme/submission-1")) +for finding in report.errors: + print(finding.rule, finding.message) +``` + +```sh +uv run inference-endpoint-validation --submission submissions/acme/submission-1 +uv run inference-endpoint-validation path/to/2026-10-C1 --submission submissions/acme/submission-1 +``` + +The API accepts a loaded `Policy` or policy directory through its `policy` keyword. +`SubmissionChecker` stores configuration; each `run()` parses fresh evidence. +Execution classifies subjects, selects rules and exceptions, resolves prerequisites, +and invokes callable evaluator classes with effective YAML requirements. Region +and power facts use the effective rules for their selected subjects. Disabled or +failed derivations cannot satisfy dependent checks. Check selection and compliance +evaluation are separate stages. + +Checks register by `CheckKind` through `Evaluator.__init_subclass__`. An evaluator +receives a `PlannedCheck` and parsed artifacts and returns explicit findings. It does +not identify behavior by a rule's name. Adding another check of an existing kind +requires a YAML definition; adding a new operation requires its evaluator and +versioned parser contract. Evidence models support safe repeated validation; +compliance findings come from the selected evaluators. + +Exit code 1 indicates errors, or warnings with `--strict`. Blocked mandatory checks +are errors and cannot produce a passing submission. Invalid artifacts, unavailable +required catalogs, unsupported operations, and missing approval data fail visibly. +The CLI report includes cohort version, revision, and policy digest. + +Seed values come from the bundled published cohort catalog and are not duplicated +in validation policies. Approved speculative decoding heads come from the selected +cohort policy. Submission validation accepts no catalog path or environment +overrides. Client revision approval requires the +`git_sha` in each point's `result_summary.json` to exactly match an entry in +`approved_client_revisions`. No Git checkout or network access is required. +An empty approval list blocks the check. Checkpoint approvals require the model, +repository, and full revision to match. The bundled client approval list requires +published values before that rule can approve submissions. + +## Policy bundles + +The cohort directory contains: + +| File | Contents | +| ---------------------- | ---------------------------------------------------------------------------- | +| catalog.yaml | Models, datasets, thresholds, enrollment, exceptions and catalog diagnostics | +| submission_checks.yaml | Submission and implementation checks | +| system_checks.yaml | System checks | +| curve_checks.yaml | Model curve and collection checks | +| point_checks.yaml | Measurement point checks | + +All files declare the same cohort `version` and positive integer `revision`. +The directory name matches the cohort. There is no top-level `kind` or +`schema_version`. `load_policy()` rejects incomplete bundles, duplicate keys, +noncanonical identifiers, unknown vocabulary, wrong scopes, and unresolved +references. Rules and catalogs are immutable; source hashes identify the bundle. +Datasets declare their exact `sample_count`, with `is_legacy` defaulting to false +and `sample_unit` defaulting to sample. + +`BundleParser` is a callable abstract class registered through `__init_subclass__`. +Each parser declares a cohort and inclusive revision interval. +`schemas/requirements_v1.py` defines a typed contract for each check kind, +including strict flags, numeric bounds, nested operands, and operation-specific +required fields. Base rules and fully merged overrides must satisfy those +contracts before planning. `operations.py` shares operation enums between +contracts and evaluators. Overlapping intervals +and unsupported releases fail explicitly. Revision 1 is supported. + +## Classification and planning + +Each input file is read and parsed once. `ParsedArtifact.from_json` retains the +typed value, supplied field names, and structural errors; it discards the input +document. Supplied field names distinguish omissions from schema defaults. +Reported metric aliases remain separate from calculated metrics. Accuracy scores +accept finite numbers and finite numeric strings; booleans, malformed values, +and explicitly empty scores produce structural errors. Supplied non-null decode +head declarations select approval checks even when identity is incomplete. +Cooling uses the shared `Cooling` enum, including `mixed`, and requires a +matching policy overhead. + +`PointArtifacts.from_evidence` constructs a complete point with parsed evidence +and a required classification. Calculated point power lives in `PointDerived`. +Submission indexes, loaded catalogs, and calculated collection data live in +`ArtifactIndex`, `CatalogEvidence`, and `DerivedArtifacts`, respectively. + +Artifact adapters create typed `Context` objects using the client's +`LoadPatternType`. `None` denotes unknown classification; empty fact sets and false +represent known absence. Available dependencies must have passed parsing or +computation. There is no deployment-scenario classification. + +Rules declare inline `applies_to`, `unless`, and `requires`. Different predicate +fields are ANDed; enum lists are alternatives. Exclusion takes precedence over +missing evidence. Unknown classification or missing prerequisites block the check. +Model-bound checks require enrollment; independent artifact checks remain eligible. + +Pareto curves contain measurement points for one model on one system. Their +classification objects contain point members. Selection filters those members, and model +or load-pattern exceptions partition them. Evaluators consume only the planned +`selected_members`. Members inherit shared model/division identity while unknown +point-specific facts remain unknown. Two exceptions matching the same member are +an error. + +```sh +uv run inference-endpoint-validation +uv run inference-endpoint-validation --submission ./submission +uv run pytest tests/unit/validation -q --no-cov +``` + +`--submission DIRECTORY` is the submission validation entry point: it reads result +artifacts, classifies runs automatically, selects checks, and executes them. Its +output contains pass/fail findings. No classification JSON is required. + +Running without `--submission` inspects the policy bundle and returns its version, +revision, rule count, and source hashes. Inspection returns exit code 0 when it +successfully produces output. Submission execution obtains classification from +artifacts and reports validation findings. +[Policy coverage](README.md) lists the supported check definitions. + +Metrics snapshots use one typed msgspec decoder. Wire verdicts use canonical enum +values; malformed or noncanonical frames follow the codec's decode-error handler. + +## Module ownership + +`evidence/` defines artifact schemas and loaders. It normalizes native field +representations and reports structural parsing errors; compliance checks belong +in `checks/`. `results.py` defines findings and submission reports using the shared +severity enum. `catalogs/` holds the published seed catalog and its loader, while +`cohorts.py` supplies publication-calendar arithmetic. + +`power/models.py` defines descriptor schemas and calculation results. +`power/calculation.py` computes power from an explicit descriptor and effective +cohort policy values. The calculator keeps policy state separate from artifact +models. The power evaluators select subjects and turn calculation findings into +validation results. + +Lower-priority CLI work is tracked in [Follow-ups](follow-ups.md). diff --git a/pyproject.toml b/pyproject.toml index ae6a31f5e..f2541b78d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -176,6 +176,7 @@ bfcl = [ [project.scripts] inference-endpoint = "inference_endpoint.main:run" +inference-endpoint-validation = "inference_endpoint.validation.__main__:main" [project.urls] Homepage = "https://github.com/mlperf/inference-endpoint" diff --git a/src/inference_endpoint/validation/__main__.py b/src/inference_endpoint/validation/__main__.py new file mode 100644 index 000000000..aef372ae8 --- /dev/null +++ b/src/inference_endpoint/validation/__main__.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Inspect cohort policies or validate submission artifacts.""" + +import argparse +import json +from pathlib import Path + +from . import ( + bundled_policy_path, + load_policy, + validate_submission, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Validate submission artifacts or inspect a cohort policy bundle.", + epilog=( + "Examples:\n" + " inference-endpoint-validation --submission ./submission\n" + " Read artifacts, classify runs, and execute validation checks.\n" + " inference-endpoint-validation\n" + " Inspect the selected policy bundle." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("policy", nargs="?", type=Path, default=bundled_policy_path()) + parser.add_argument( + "--submission", + type=Path, + metavar="DIRECTORY", + help=( + "Read a submission directory, automatically classify its runs, and " + "execute checks; output validation findings" + ), + ) + parser.add_argument( + "--strict", action="store_true", help="Treat validation warnings as failures" + ) + arguments = parser.parse_args() + try: + if arguments.submission: + policy = load_policy(arguments.policy) + report = validate_submission( + arguments.submission, + policy=policy, + ) + output = report.model_dump(mode="json") + output["engine"] = "policy" + output["version"] = policy.release.version + output["revision"] = policy.release.revision + output["policy_digest"] = policy.digest + print(json.dumps(output, indent=2)) + return 1 if report.errors or (arguments.strict and report.warnings) else 0 + if arguments.strict: + raise ValueError("Execution options require --submission") + policy = load_policy(arguments.policy) + output = { + "version": policy.release.version, + "revision": policy.release.revision, + "policy_digest": policy.digest, + "rules": len(policy.checks), + "files": { + file.value: digest for file, digest in policy.source_digests.items() + }, + } + print(json.dumps(output, indent=2)) + except (OSError, ValueError) as error: + parser.exit(2, f"{error}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/validation/test_cli.py b/tests/unit/validation/test_cli.py new file mode 100644 index 000000000..a7c8e2826 --- /dev/null +++ b/tests/unit/validation/test_cli.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Policy inspection and submission validation CLI contracts.""" + +import json +import sys + +import pytest + +from inference_endpoint.validation.__main__ import main + +pytestmark = pytest.mark.unit + + +def test_inspect_bundle(monkeypatch, capsys): + monkeypatch.setattr(sys, "argv", ["validation"]) + assert main() == 0 + output = json.loads(capsys.readouterr().out) + assert output["rules"] == 89 + assert len(output["files"]) == 5 + assert "checks" not in output + + +def test_submission_command_executes_checks(tmp_path, monkeypatch, capsys): + missing = tmp_path / "missing" + monkeypatch.setattr(sys, "argv", ["validation", "--submission", str(missing)]) + assert main() == 1 + output = json.loads(capsys.readouterr().out) + assert output["engine"] == "policy" + assert not output["passed"] + assert any(result["rule"] == "path-exists" for result in output["errors"]) + + +def test_submission_execution_rejects_invalid_policy_bundle( + tmp_path, monkeypatch, capsys +): + monkeypatch.setattr( + sys, "argv", ["validation", str(tmp_path), "--submission", str(tmp_path)] + ) + with pytest.raises(SystemExit) as error: + main() + assert error.value.code == 2 + assert "policy files" in capsys.readouterr().err + + +@pytest.mark.parametrize("option", ["--seed-sets", "--approved-sped-decode-heads"]) +def test_cli_rejects_catalog_overrides(option, tmp_path, monkeypatch, capsys): + monkeypatch.setattr(sys, "argv", ["validation", option, str(tmp_path)]) + with pytest.raises(SystemExit) as error: + main() + assert error.value.code == 2 + assert "unrecognized arguments" in capsys.readouterr().err From a2b97018c0db27c7497fcbac4ccb38d388f9686b Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 01:43:01 -0700 Subject: [PATCH 08/18] docs(validation): clarify conditions and evidence as bullet points --- docs/validation/README.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/validation/README.md b/docs/validation/README.md index cdf3dce00..aabe72e82 100644 --- a/docs/validation/README.md +++ b/docs/validation/README.md @@ -18,14 +18,20 @@ names and references are validated by the versioned policy parser. ## Conditions and evidence -Each check declares inline inclusion/exclusion predicates and dependencies. -Different fields are ANDed; enum lists are alternatives. Exclusion wins. -Unknown classification blocks selection. Empty fact sets represent known absence. -Curve member exceptions produce distinct plans with selected member IDs. -Overlapping exceptions on one member fail explicitly. Point-specific unknown -facts remain unknown. Independent checks stay planned when another artifact fails. -Datasets normalize is_legacy to false and sample_unit to sample when omitted. -Seeds come from the bundled published cohort catalog. +- Each check declares when it applies, when it is excluded, and which evidence it + requires. +- All condition fields must match. Within an enum list, any listed value can match. + A matching exclusion prevents the check from running. +- Missing classification information blocks checks that need it. An empty set + means the information is known and none of the listed facts are present. +- Exceptions for points within a Pareto curve create separate plans for the + affected points. Overlapping exceptions for the same point cause an error. +- Each point retains its own classification information, including unknown values. +- An invalid artifact blocks checks that depend on it; independent checks can + still run. +- Dataset entries default to `is_legacy: false` and `sample_unit: sample` when those + fields are omitted. +- Seeds come from the bundled published cohort catalog. ## Unresolved policy inputs From aeaa68f9eb16cfc6d45012413a3b8e648817c220 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 01:53:55 -0700 Subject: [PATCH 09/18] docs(validation): explain evaluator kinds and group rules by scope --- docs/validation/README.md | 267 +++++++++++++++++++++++++------------- 1 file changed, 176 insertions(+), 91 deletions(-) diff --git a/docs/validation/README.md b/docs/validation/README.md index aabe72e82..c378e8d24 100644 --- a/docs/validation/README.md +++ b/docs/validation/README.md @@ -40,96 +40,181 @@ supplied before client revision approval can pass. Speculative-head approval publication cohorts are unspecified. The policy catalog is proposed, not an official MLCommons publication. +## Evaluator kinds + +- A rule's YAML `kind` selects its callable evaluator class. +- The rule supplies the evidence addresses, catalogs, thresholds, and other + requirements used by that evaluator. Some kinds also use an `operation` to + select a specific behavior. +- The planner selects applicable checks and verifies prerequisites before + evaluators run. Evaluators return findings for the validation report. + +| Evaluator kind | How it works | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `accuracy_coverage` | Checks that accuracy results cover the required concurrency bands and offline points. | +| `accuracy_gate` | Compares model accuracy scores, issued accuracy sample counts, or output-length statistics with policy thresholds; supports per-point and aggregated results. | +| `accuracy_presence` | Checks that enough model curves contain standalone or embedded accuracy results. | +| `artifact_binding` | Matches a submitted client SHA or checkpoint repository and revision against approved catalog entries. | +| `artifact_schema` | Checks that an artifact parsed successfully into its typed model and, when required, is nonempty. | +| `catalog_integrity` | Checks that a required seed or speculative-head catalog loaded successfully and is available. | +| `cohort_identifier` | Checks the target cohort string has the expected year, month, and C0/C1 format. | +| `collection_size` | Checks that the number of points or other collection members falls within the configured limits. | +| `comparison` | Compares two artifact values or constants using equality, greater-than, or greater-than-or-equal. | +| `consistency` | Checks equality between two values or across declarations in a collection, optionally excluding specified fields. | +| `coverage` | Counts performance points in a concurrency band and checks the required minimum. | +| `dataset_count` | Checks that a declared count equals the dataset sample count or is a positive multiple of it. | +| `dataset_minimum` | Checks that completed samples meet the threshold for the point's dataset. | +| `derived_metric` | Recalculates throughput, interactivity, utilization, or throughput per kW and compares it with the reported value using a tolerance. | +| `disclosure` | Checks that required configuration disclosure fields are present and nonempty. | +| `drafter_binding` | Checks speculative decoding head approval by model, repository, and revision, or checks approval lead time. | +| `duration` | Checks runtime against the threshold for the concurrency band, using the configured precedence for window and whole-run durations. | +| `field_constraints` | Checks that configured fields are absent or exactly equal to prescribed values, including model-specific constraints. | +| `issuance` | Checks that the load pattern is allowed and, when required, concurrency is positive. | +| `membership` | Checks that a value belongs to a configured catalog of allowed values. | +| `numeric_validity` | Checks numeric type and configured requirements for presence, finiteness, and positivity. | +| `offline` | Checks required offline points, elected-offline concurrency, or throughput ordering relative to other points. | +| `path_resolution` | Resolves shared source and documentation paths and checks existence, directory type, and allowed traversal or symlink behavior. | +| `power` | Checks system power descriptors and sources, reports estimated power, calculates per-point power from engaged nodes, or checks accelerator capacity and engagement. | +| `presence` | Checks that required files, directories, declarations, or collection members exist. | +| `region_basis` | Reports the minimum concurrency used to derive regions and checks that readable concurrency values are available. | +| `region_boundaries` | Checks the minimum and maximum concurrency requirements and that region boundaries were computed. | +| `region_placement` | Checks that a point lies in an allowed computed region or that its declared region matches. | +| `report` | Includes a resolved artifact value in the report without applying a pass/fail constraint. | +| `seed_binding` | Checks published seed-set membership, runtime seed values, adoption windows, or legacy seed field usage. | +| `steady_state` | Checks status and verdict vocabulary, window consistency, or reporting basis; reports drift and unavailable official windows. | +| `warmup` | Checks that warmup salt is disabled or log retention is declared, with configured exemptions for disabled warmup. | + ## Definitions -| Rule ID | Scope | Evaluator kind | Policy file | -| ---------------------------------- | -------------- | ------------------- | ------------------------ | -| `accuracy-coverage` | curve | `accuracy_coverage` | `curve_checks.yaml` | -| `accuracy-gate` | curve | `accuracy_gate` | `curve_checks.yaml` | -| `accuracy-present` | submission | `accuracy_presence` | `submission_checks.yaml` | -| `accuracy-sample-count` | curve | `accuracy_gate` | `curve_checks.yaml` | -| `accuracy-valid` | point | `artifact_schema` | `point_checks.yaml` | -| `agentic-accuracy` | curve | `accuracy_gate` | `curve_checks.yaml` | -| `agentic-accuracy-inline` | curve | `accuracy_gate` | `curve_checks.yaml` | -| `agentic-accuracy-swebench` | curve | `accuracy_gate` | `curve_checks.yaml` | -| `agentic-metric-consistency` | point | `derived_metric` | `point_checks.yaml` | -| `agentic-osl-range` | curve | `accuracy_gate` | `curve_checks.yaml` | -| `agentic-trajectory-count` | point | `dataset_count` | `point_checks.yaml` | -| `approved-checkpoint` | point | `artifact_binding` | `point_checks.yaml` | -| `approved-drafter` | point | `drafter_binding` | `point_checks.yaml` | -| `benchmark-model-dir` | system | `presence` | `system_checks.yaml` | -| `benchmark-type-consistency` | curve | `consistency` | `curve_checks.yaml` | -| `concurrency-in-range` | point | `region_placement` | `point_checks.yaml` | -| `config-consistency-dataset` | curve | `consistency` | `curve_checks.yaml` | -| `config-consistency-model` | curve | `consistency` | `curve_checks.yaml` | -| `drafter-approval-lead-time` | point | `drafter_binding` | `point_checks.yaml` | -| `drafter-list-registry` | validator | `catalog_integrity` | `catalog.yaml` | -| `endpoints-client-sha` | point | `artifact_binding` | `point_checks.yaml` | -| `high-concurrency-coverage` | curve | `coverage` | `curve_checks.yaml` | -| `load-pattern` | point | `issuance` | `point_checks.yaml` | -| `low-concurrency-coverage` | curve | `coverage` | `curve_checks.yaml` | -| `max-concurrency-declared` | curve | `report` | `curve_checks.yaml` | -| `maximal-engagement` | point | `power` | `point_checks.yaml` | -| `measurement-points-present` | curve | `presence` | `curve_checks.yaml` | -| `med-concurrency-coverage` | curve | `coverage` | `curve_checks.yaml` | -| `metric-consistency-accounting` | point | `comparison` | `point_checks.yaml` | -| `metric-consistency-duration` | point | `comparison` | `point_checks.yaml` | -| `metric-consistency-output-tokens` | point | `comparison` | `point_checks.yaml` | -| `metric-consistency-system-tps` | point | `derived_metric` | `point_checks.yaml` | -| `metric-consistency-tpot-p90` | point | `numeric_validity` | `point_checks.yaml` | -| `metric-consistency-tps-per-kw` | point | `derived_metric` | `point_checks.yaml` | -| `metric-consistency-tps-per-user` | point | `derived_metric` | `point_checks.yaml` | -| `min-query-count` | point | `dataset_minimum` | `point_checks.yaml` | -| `model-name-consistency` | curve | `consistency` | `curve_checks.yaml` | -| `model-name-valid` | curve | `membership` | `curve_checks.yaml` | -| `nodes-used` | point | `power` | `point_checks.yaml` | -| `offline-declared` | point | `membership` | `point_checks.yaml` | -| `offline-ordering` | curve | `offline` | `curve_checks.yaml` | -| `offline-point-present` | curve | `offline` | `curve_checks.yaml` | -| `path-exists` | submission | `presence` | `submission_checks.yaml` | -| `point-cap` | curve | `collection_size` | `curve_checks.yaml` | -| `point-config-valid` | point | `artifact_schema` | `point_checks.yaml` | -| `point-count` | curve | `collection_size` | `curve_checks.yaml` | -| `point-dirname-concurrency` | point | `consistency` | `point_checks.yaml` | -| `point-dirs` | curve | `presence` | `curve_checks.yaml` | -| `point-disclosure-complete` | point | `disclosure` | `point_checks.yaml` | -| `point-duration` | point | `duration` | `point_checks.yaml` | -| `point-power` | point | `power` | `point_checks.yaml` | -| `point-rules-skipped` | point | `report` | `point_checks.yaml` | -| `power-descriptor` | system | `power` | `system_checks.yaml` | -| `power-estimated` | system | `power` | `system_checks.yaml` | -| `region-basis` | curve | `region_basis` | `curve_checks.yaml` | -| `region-computation` | curve | `region_boundaries` | `curve_checks.yaml` | -| `region-declared` | point | `membership` | `point_checks.yaml` | -| `region-placement` | point | `region_placement` | `point_checks.yaml` | -| `required-dir` | submission | `presence` | `submission_checks.yaml` | -| `result-file-valid` | point | `artifact_schema` | `point_checks.yaml` | -| `result-summary-present` | point | `presence` | `point_checks.yaml` | -| `seed-config-legacy` | point | `seed_binding` | `point_checks.yaml` | -| `seed-runtime-match` | point | `seed_binding` | `point_checks.yaml` | -| `seed-set-adoption` | point | `seed_binding` | `point_checks.yaml` | -| `seed-set-consistency` | curve | `consistency` | `curve_checks.yaml` | -| `seed-set-membership` | point | `seed_binding` | `point_checks.yaml` | -| `seed-set-registry` | validator | `catalog_integrity` | `catalog.yaml` | -| `shared-path-resolution` | point | `path_resolution` | `point_checks.yaml` | -| `src-dir` | submission | `presence` | `submission_checks.yaml` | -| `src-readme` | implementation | `presence` | `submission_checks.yaml` | -| `steady-state-basis` | point | `steady_state` | `point_checks.yaml` | -| `steady-state-consistency` | point | `steady_state` | `point_checks.yaml` | -| `steady-state-valid` | point | `steady_state` | `point_checks.yaml` | -| `streaming-config` | point | `report` | `point_checks.yaml` | -| `submission-flags` | point | `field_constraints` | `point_checks.yaml` | -| `swebench-instance-count` | point | `dataset_count` | `point_checks.yaml` | -| `swebench-template` | point | `field_constraints` | `point_checks.yaml` | -| `system-description-across-curves` | system | `consistency` | `system_checks.yaml` | -| `system-description-consistency` | curve | `consistency` | `curve_checks.yaml` | -| `system-description-present` | point | `presence` | `point_checks.yaml` | -| `system-description-valid` | point | `artifact_schema` | `point_checks.yaml` | -| `system-name-consistency` | submission | `consistency` | `submission_checks.yaml` | -| `system-results-dir` | submission | `presence` | `submission_checks.yaml` | -| `target-cohort` | point | `cohort_identifier` | `point_checks.yaml` | -| `tps-utilization` | curve | `derived_metric` | `curve_checks.yaml` | -| `ultra-low-concurrency-coverage` | curve | `coverage` | `curve_checks.yaml` | -| `warmup-logs-retained` | point | `warmup` | `point_checks.yaml` | -| `warmup-present` | point | `presence` | `point_checks.yaml` | -| `warmup-salt` | point | `warmup` | `point_checks.yaml` | +Rules are grouped by scope; each group lists its policy file once. + +### Submission + +Policy file: `submission_checks.yaml`. + +| Rule ID | Evaluator kind | +| ------------------------- | ------------------- | +| `accuracy-present` | `accuracy_presence` | +| `path-exists` | `presence` | +| `required-dir` | `presence` | +| `src-dir` | `presence` | +| `system-name-consistency` | `consistency` | +| `system-results-dir` | `presence` | + +### Implementation + +Policy file: `submission_checks.yaml`. + +| Rule ID | Evaluator kind | +| ------------ | -------------- | +| `src-readme` | `presence` | + +### System + +Policy file: `system_checks.yaml`. + +| Rule ID | Evaluator kind | +| ---------------------------------- | -------------- | +| `benchmark-model-dir` | `presence` | +| `power-descriptor` | `power` | +| `power-estimated` | `power` | +| `system-description-across-curves` | `consistency` | + +### Pareto curve + +Policy file: `curve_checks.yaml`. + +| Rule ID | Evaluator kind | +| -------------------------------- | ------------------- | +| `accuracy-coverage` | `accuracy_coverage` | +| `accuracy-gate` | `accuracy_gate` | +| `accuracy-sample-count` | `accuracy_gate` | +| `agentic-accuracy` | `accuracy_gate` | +| `agentic-accuracy-inline` | `accuracy_gate` | +| `agentic-accuracy-swebench` | `accuracy_gate` | +| `agentic-osl-range` | `accuracy_gate` | +| `benchmark-type-consistency` | `consistency` | +| `config-consistency-dataset` | `consistency` | +| `config-consistency-model` | `consistency` | +| `high-concurrency-coverage` | `coverage` | +| `low-concurrency-coverage` | `coverage` | +| `max-concurrency-declared` | `report` | +| `measurement-points-present` | `presence` | +| `med-concurrency-coverage` | `coverage` | +| `model-name-consistency` | `consistency` | +| `model-name-valid` | `membership` | +| `offline-ordering` | `offline` | +| `offline-point-present` | `offline` | +| `point-cap` | `collection_size` | +| `point-count` | `collection_size` | +| `point-dirs` | `presence` | +| `region-basis` | `region_basis` | +| `region-computation` | `region_boundaries` | +| `seed-set-consistency` | `consistency` | +| `system-description-consistency` | `consistency` | +| `tps-utilization` | `derived_metric` | +| `ultra-low-concurrency-coverage` | `coverage` | + +### Point + +Policy file: `point_checks.yaml`. + +| Rule ID | Evaluator kind | +| ---------------------------------- | ------------------- | +| `accuracy-valid` | `artifact_schema` | +| `agentic-metric-consistency` | `derived_metric` | +| `agentic-trajectory-count` | `dataset_count` | +| `approved-checkpoint` | `artifact_binding` | +| `approved-drafter` | `drafter_binding` | +| `concurrency-in-range` | `region_placement` | +| `drafter-approval-lead-time` | `drafter_binding` | +| `endpoints-client-sha` | `artifact_binding` | +| `load-pattern` | `issuance` | +| `maximal-engagement` | `power` | +| `metric-consistency-accounting` | `comparison` | +| `metric-consistency-duration` | `comparison` | +| `metric-consistency-output-tokens` | `comparison` | +| `metric-consistency-system-tps` | `derived_metric` | +| `metric-consistency-tpot-p90` | `numeric_validity` | +| `metric-consistency-tps-per-kw` | `derived_metric` | +| `metric-consistency-tps-per-user` | `derived_metric` | +| `min-query-count` | `dataset_minimum` | +| `nodes-used` | `power` | +| `offline-declared` | `membership` | +| `point-config-valid` | `artifact_schema` | +| `point-dirname-concurrency` | `consistency` | +| `point-disclosure-complete` | `disclosure` | +| `point-duration` | `duration` | +| `point-power` | `power` | +| `point-rules-skipped` | `report` | +| `region-declared` | `membership` | +| `region-placement` | `region_placement` | +| `result-file-valid` | `artifact_schema` | +| `result-summary-present` | `presence` | +| `seed-config-legacy` | `seed_binding` | +| `seed-runtime-match` | `seed_binding` | +| `seed-set-adoption` | `seed_binding` | +| `seed-set-membership` | `seed_binding` | +| `shared-path-resolution` | `path_resolution` | +| `steady-state-basis` | `steady_state` | +| `steady-state-consistency` | `steady_state` | +| `steady-state-valid` | `steady_state` | +| `streaming-config` | `report` | +| `submission-flags` | `field_constraints` | +| `swebench-instance-count` | `dataset_count` | +| `swebench-template` | `field_constraints` | +| `system-description-present` | `presence` | +| `system-description-valid` | `artifact_schema` | +| `target-cohort` | `cohort_identifier` | +| `warmup-logs-retained` | `warmup` | +| `warmup-present` | `presence` | +| `warmup-salt` | `warmup` | + +### Validator catalogs + +Policy file: `catalog.yaml`. + +| Rule ID | Evaluator kind | +| ----------------------- | ------------------- | +| `drafter-list-registry` | `catalog_integrity` | +| `seed-set-registry` | `catalog_integrity` | From 7eab5c1fcce17fc23538541a90804ad58cfd6036 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 02:06:57 -0700 Subject: [PATCH 10/18] docs(validation): group evaluator kinds and their shared operations --- docs/validation/README.md | 130 ++++++++++++++++++++++++++------------ 1 file changed, 90 insertions(+), 40 deletions(-) diff --git a/docs/validation/README.md b/docs/validation/README.md index c378e8d24..af19304d8 100644 --- a/docs/validation/README.md +++ b/docs/validation/README.md @@ -42,47 +42,97 @@ The policy catalog is proposed, not an official MLCommons publication. ## Evaluator kinds -- A rule's YAML `kind` selects its callable evaluator class. -- The rule supplies the evidence addresses, catalogs, thresholds, and other - requirements used by that evaluator. Some kinds also use an `operation` to - select a specific behavior. +- A rule's YAML `kind` selects a callable evaluator class; its other fields supply + evidence addresses, catalogs, thresholds, and requirements. +- Each kind below has its own registry entry. The groups organize related checks; + they do not represent class inheritance. +- Nested operations under a kind share its evaluator class. For example, client + SHA and checkpoint approval both use `artifact_binding`. - The planner selects applicable checks and verifies prerequisites before - evaluators run. Evaluators return findings for the validation report. - -| Evaluator kind | How it works | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `accuracy_coverage` | Checks that accuracy results cover the required concurrency bands and offline points. | -| `accuracy_gate` | Compares model accuracy scores, issued accuracy sample counts, or output-length statistics with policy thresholds; supports per-point and aggregated results. | -| `accuracy_presence` | Checks that enough model curves contain standalone or embedded accuracy results. | -| `artifact_binding` | Matches a submitted client SHA or checkpoint repository and revision against approved catalog entries. | -| `artifact_schema` | Checks that an artifact parsed successfully into its typed model and, when required, is nonempty. | -| `catalog_integrity` | Checks that a required seed or speculative-head catalog loaded successfully and is available. | -| `cohort_identifier` | Checks the target cohort string has the expected year, month, and C0/C1 format. | -| `collection_size` | Checks that the number of points or other collection members falls within the configured limits. | -| `comparison` | Compares two artifact values or constants using equality, greater-than, or greater-than-or-equal. | -| `consistency` | Checks equality between two values or across declarations in a collection, optionally excluding specified fields. | -| `coverage` | Counts performance points in a concurrency band and checks the required minimum. | -| `dataset_count` | Checks that a declared count equals the dataset sample count or is a positive multiple of it. | -| `dataset_minimum` | Checks that completed samples meet the threshold for the point's dataset. | -| `derived_metric` | Recalculates throughput, interactivity, utilization, or throughput per kW and compares it with the reported value using a tolerance. | -| `disclosure` | Checks that required configuration disclosure fields are present and nonempty. | -| `drafter_binding` | Checks speculative decoding head approval by model, repository, and revision, or checks approval lead time. | -| `duration` | Checks runtime against the threshold for the concurrency band, using the configured precedence for window and whole-run durations. | -| `field_constraints` | Checks that configured fields are absent or exactly equal to prescribed values, including model-specific constraints. | -| `issuance` | Checks that the load pattern is allowed and, when required, concurrency is positive. | -| `membership` | Checks that a value belongs to a configured catalog of allowed values. | -| `numeric_validity` | Checks numeric type and configured requirements for presence, finiteness, and positivity. | -| `offline` | Checks required offline points, elected-offline concurrency, or throughput ordering relative to other points. | -| `path_resolution` | Resolves shared source and documentation paths and checks existence, directory type, and allowed traversal or symlink behavior. | -| `power` | Checks system power descriptors and sources, reports estimated power, calculates per-point power from engaged nodes, or checks accelerator capacity and engagement. | -| `presence` | Checks that required files, directories, declarations, or collection members exist. | -| `region_basis` | Reports the minimum concurrency used to derive regions and checks that readable concurrency values are available. | -| `region_boundaries` | Checks the minimum and maximum concurrency requirements and that region boundaries were computed. | -| `region_placement` | Checks that a point lies in an allowed computed region or that its declared region matches. | -| `report` | Includes a resolved artifact value in the report without applying a pass/fail constraint. | -| `seed_binding` | Checks published seed-set membership, runtime seed values, adoption windows, or legacy seed field usage. | -| `steady_state` | Checks status and verdict vocabulary, window consistency, or reporting basis; reports drift and unavailable official windows. | -| `warmup` | Checks that warmup salt is disabled or log retention is declared, with configured exemptions for disabled warmup. | + evaluators return findings for the report. + +- **Artifact structure and required evidence** + - `presence`: requires files, directories, declarations, or collection members. + - `artifact_schema`: requires successful typed parsing and, when configured, + nonempty contents. + - `disclosure`: requires nonempty configuration fields. + - `path_resolution`: resolves shared source and documentation paths and checks + existence, directory type, and allowed traversal or symlink behavior. + - `catalog_integrity`: requires a successfully loaded seed or speculative-head + catalog. +- **Values and collections** + - `membership`: checks a value against an allowed catalog. + - `field_constraints`: requires fields to be absent or exactly equal to + prescribed values, including model-specific constraints. + - `numeric_validity`: checks numeric type, presence, finiteness, and positivity. + - `comparison`: compares artifact values or constants using equality, + greater-than, or greater-than-or-equal. + - `consistency`: checks equality between two values or across a collection of + declarations, optionally excluding specified fields. + - `collection_size`: checks the number of members against configured limits. + - `report`: includes a resolved value in the report without a pass/fail + constraint. +- **Run classification and Pareto curves** + - `cohort_identifier`: checks the cohort string's year, month, and C0/C1 format. + - `issuance`: checks allowed load patterns and, when required, positive + concurrency. + - `region_basis`: reports the minimum concurrency used to derive regions and + requires readable concurrency values. + - `region_boundaries`: checks minimum and maximum concurrency requirements and + that region boundaries were computed. + - `region_placement`: + - Checks that a point lies within an allowed computed region. + - Checks that the declared region matches the computed region. + - `coverage`: counts performance points in a concurrency band and checks the + required minimum. + - `offline`: + - Checks required offline points and elected-offline concurrency. + - Checks offline throughput ordering relative to other points. +- **Datasets and accuracy** + - `dataset_minimum`: compares completed samples with the dataset's threshold. + - `dataset_count`: requires the dataset's exact sample count or a positive + multiple of it. + - `accuracy_presence`: requires accuracy results in enough model curves. + - `accuracy_coverage`: requires accuracy results across concurrency bands and + offline points. + - `accuracy_gate`: + - Checks that an agentic accuracy profile is available. + - Compares single-turn scores and issued accuracy sample counts with model + requirements. + - Compares agentic scores per point or as a mean of concurrency-band means. + - Checks output-length statistics against a model-specific range. +- **Approved artifacts and seeds** + - `artifact_binding`: + - Matches the submitted client SHA against the approved list. + - Matches checkpoint repository and revision against the model's approvals. + - `drafter_binding`: + - Matches a speculative decoding head by model, repository, and revision. + - Checks its approval lead time in cohorts. + - `seed_binding`: + - Checks published seed-set membership. + - Compares runtime seed values with the published set. + - Checks the seed set's adoption window. + - Reports legacy seed field usage. +- **Measurements and runtime** + - `duration`: compares runtime with the concurrency band's threshold, using the + configured precedence for window and whole-run durations. + - `derived_metric`: recalculates a metric and compares it with the reported + value using a tolerance. Operations cover throughput, interactivity, + utilization, and throughput per kW. + - `steady_state`: + - Checks status and verdict vocabulary. + - Checks window and reported super-pass consistency. + - Checks reporting basis and reports drift or unavailable official windows. + - `warmup`: + - Requires disabled warmup salt. + - Checks declared log retention, with configured exemptions for disabled + warmup. +- **Power** + - `power`: + - Checks system power descriptors and sources. + - Reports estimated power that uses component defaults. + - Calculates per-point power from engaged nodes. + - Checks declared accelerator capacity and maximal engagement. ## Definitions From ff433f6baea3758e383a208fb293d0ac3cce08e0 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 03:35:37 -0700 Subject: [PATCH 11/18] refactor(validation): consolidate count checks and clarify rule names --- docs/validation/README.md | 21 ++- .../validation/checks/datasets.py | 77 +++++---- .../validation/operations.py | 3 +- .../policies/2026-10-C1/point_checks.yaml | 12 +- .../validation/schemas/requirements_v1.py | 30 ++-- .../validation/vocabulary.py | 3 +- .../validation/policy-fingerprints.json | 6 +- tests/unit/validation/test_counts.py | 161 ++++++++++++++++++ tests/unit/validation/test_engine.py | 12 +- tests/unit/validation/test_loader.py | 6 +- tests/unit/validation/test_planner.py | 18 +- 11 files changed, 256 insertions(+), 93 deletions(-) create mode 100644 tests/unit/validation/test_counts.py diff --git a/docs/validation/README.md b/docs/validation/README.md index af19304d8..a4e3e8df3 100644 --- a/docs/validation/README.md +++ b/docs/validation/README.md @@ -24,8 +24,9 @@ names and references are validated by the versioned policy parser. A matching exclusion prevents the check from running. - Missing classification information blocks checks that need it. An empty set means the information is known and none of the listed facts are present. -- Exceptions for points within a Pareto curve create separate plans for the - affected points. Overlapping exceptions for the same point cause an error. +- Exceptions can change or disable a check for specific points within a Pareto + curve. Other points are checked using the original requirements. Overlapping + exceptions for the same point cause an error. - Each point retains its own classification information, including unknown values. - An invalid artifact blocks checks that depend on it; independent checks can still run. @@ -89,9 +90,13 @@ The policy catalog is proposed, not an official MLCommons publication. - Checks required offline points and elected-offline concurrency. - Checks offline throughput ordering relative to other points. - **Datasets and accuracy** - - `dataset_minimum`: compares completed samples with the dataset's threshold. - - `dataset_count`: requires the dataset's exact sample count or a positive - multiple of it. + - `count`: compares a reported or configured count with a catalog reference. + - `min-completed-samples`: completed samples must meet or exceed the count for + the point's configured dataset. + - `trajectories-to-issue`: the configured issuance target must be a positive + multiple of the trajectory count. + - `swebench-instance-count`: declared and evaluated instance counts must each + equal the catalog count. - `accuracy_presence`: requires accuracy results in enough model curves. - `accuracy_coverage`: requires accuracy results across concurrency bands and offline points. @@ -213,7 +218,6 @@ Policy file: `point_checks.yaml`. | ---------------------------------- | ------------------- | | `accuracy-valid` | `artifact_schema` | | `agentic-metric-consistency` | `derived_metric` | -| `agentic-trajectory-count` | `dataset_count` | | `approved-checkpoint` | `artifact_binding` | | `approved-drafter` | `drafter_binding` | | `concurrency-in-range` | `region_placement` | @@ -228,7 +232,7 @@ Policy file: `point_checks.yaml`. | `metric-consistency-tpot-p90` | `numeric_validity` | | `metric-consistency-tps-per-kw` | `derived_metric` | | `metric-consistency-tps-per-user` | `derived_metric` | -| `min-query-count` | `dataset_minimum` | +| `min-completed-samples` | `count` | | `nodes-used` | `power` | | `offline-declared` | `membership` | | `point-config-valid` | `artifact_schema` | @@ -251,11 +255,12 @@ Policy file: `point_checks.yaml`. | `steady-state-valid` | `steady_state` | | `streaming-config` | `report` | | `submission-flags` | `field_constraints` | -| `swebench-instance-count` | `dataset_count` | +| `swebench-instance-count` | `count` | | `swebench-template` | `field_constraints` | | `system-description-present` | `presence` | | `system-description-valid` | `artifact_schema` | | `target-cohort` | `cohort_identifier` | +| `trajectories-to-issue` | `count` | | `warmup-logs-retained` | `warmup` | | `warmup-present` | `presence` | | `warmup-salt` | `warmup` | diff --git a/src/inference_endpoint/validation/checks/datasets.py b/src/inference_endpoint/validation/checks/datasets.py index fe30102eb..c98c0f8f6 100644 --- a/src/inference_endpoint/validation/checks/datasets.py +++ b/src/inference_endpoint/validation/checks/datasets.py @@ -6,7 +6,7 @@ from ..artifacts import Artifacts from ..evaluator_base import Evaluator -from ..operations import DatasetCountOperator +from ..operations import CountOperator from ..outcomes import result from ..planner import PlannedCheck from ..results import CheckResult @@ -14,61 +14,60 @@ from ..vocabulary import CheckKind -class DatasetMinimum(Evaluator, kind=CheckKind.DATASET_MINIMUM): +class Count(Evaluator, kind=CheckKind.COUNT): + """Compare a reported or configured count with a catalog reference count.""" + def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: req = check.rule.requirements - point = artifacts.index.points[check.subject.id] - dataset = point.get_config("dataset") - catalog = artifacts.resolve(req["catalog"], check) or {} - entry = catalog.get(dataset, {}) - if entry.get("sample_unit", SampleUnit.SAMPLE) not in req.get( - "supported_sample_units", [SampleUnit.SAMPLE] + dataset = ( + artifacts.resolve(req["dataset_source"], check) + if "dataset_source" in req + else req["dataset"] + ) + entry = (artifacts.resolve(req["catalog"], check) or {}).get(dataset, {}) + supported_units = req.get("supported_sample_units") + if ( + supported_units is not None + and entry.get("sample_unit", SampleUnit.SAMPLE) not in supported_units ): if req.get("on_unsupported_sample_unit") == "skip": return [] return [result(check, False, "Unsupported sample unit", blocked=True)] - minimum = entry.get(req["threshold"]) - if minimum is None: + expected = entry.get(req.get("threshold", "sample_count")) + if expected is None: if req.get("on_unknown_or_null_threshold") == "skip": return [] - return [result(check, False, "Dataset threshold unavailable", blocked=True)] - actual = artifacts.resolve(req["source"], check) - return [ - result( - check, - isinstance(actual, (int, float)) - and not isinstance(actual, bool) - and actual >= minimum, - f"Completed {actual!r}; {dataset} requires {minimum}", - ) - ] - - -class DatasetCount(Evaluator, kind=CheckKind.DATASET_COUNT): - def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements - entry = (artifacts.resolve(req["catalog"], check) or {}).get(req["dataset"], {}) - expected = entry.get("sample_count") + return [result(check, False, "Reference count unavailable", blocked=True)] + if expected <= 0: + return [ + result(check, False, "Reference count must be positive", blocked=True) + ] + operation = CountOperator(req["operator"]) sources = req.get("sources") or [req["source"]] findings = [] - if expected is None or expected <= 0: - return [result(check, False, "Dataset count unavailable", blocked=True)] for source in sources: actual = artifacts.resolve(source, check) - valid = ( - isinstance(actual, int) - and not isinstance(actual, bool) - and ( - actual == expected - if req["operator"] == DatasetCountOperator.EQUAL - else actual > 0 and actual % expected == 0 + if operation is CountOperator.GREATER_THAN_OR_EQUAL: + valid = ( + isinstance(actual, (int, float)) + and not isinstance(actual, bool) + and actual >= expected + ) + else: + valid = ( + isinstance(actual, int) + and not isinstance(actual, bool) + and ( + actual == expected + if operation is CountOperator.EQUAL + else actual > 0 and actual % expected == 0 + ) ) - ) findings.append( result( check, valid, - f"{source}: {actual!r}; expected {req['operator']} {expected}", + f"{source}: {actual!r}; expected {operation.value} {expected} for {dataset}", ) ) return findings diff --git a/src/inference_endpoint/validation/operations.py b/src/inference_endpoint/validation/operations.py index 14f101bce..144919e2b 100644 --- a/src/inference_endpoint/validation/operations.py +++ b/src/inference_endpoint/validation/operations.py @@ -84,6 +84,7 @@ class DurationBasis(StrEnum): WHOLE_RUN_DURATION = "whole_run_duration" -class DatasetCountOperator(StrEnum): +class CountOperator(StrEnum): + GREATER_THAN_OR_EQUAL = "greater_than_or_equal" EQUAL = "equal" POSITIVE_MULTIPLE = "positive_multiple" diff --git a/src/inference_endpoint/validation/policies/2026-10-C1/point_checks.yaml b/src/inference_endpoint/validation/policies/2026-10-C1/point_checks.yaml index 044d23488..add565a2c 100644 --- a/src/inference_endpoint/validation/policies/2026-10-C1/point_checks.yaml +++ b/src/inference_endpoint/validation/policies/2026-10-C1/point_checks.yaml @@ -16,8 +16,8 @@ checks: target: accuracy.swe_bench.extras.swebench_template constraints_by_model: catalogs.submission_flags.swebench_templates requires: [point_config] - agentic-trajectory-count: - kind: dataset_count + trajectories-to-issue: + kind: count applies_to: { load_pattern: [agentic_inference] } dataset: agentic_combined source: point.runtime_settings.agentic_inference.num_trajectories_to_issue @@ -27,7 +27,7 @@ checks: on_missing: error requires: [point_config] swebench-instance-count: - kind: dataset_count + kind: count applies_to: { load_pattern: [agentic_inference] } dataset: swe_bench sources: @@ -203,8 +203,10 @@ checks: basis_precedence: [reported_window_issue_span, whole_run_duration] window_status_required: false on_unclassifiable_concurrency: skip - min-query-count: - kind: dataset_minimum + min-completed-samples: + kind: count + dataset_source: point.dataset + operator: greater_than_or_equal source: result_summary.n_samples_completed catalog: catalogs.datasets threshold: sample_count diff --git a/src/inference_endpoint/validation/schemas/requirements_v1.py b/src/inference_endpoint/validation/schemas/requirements_v1.py index c47d8fb73..532edaf22 100644 --- a/src/inference_endpoint/validation/schemas/requirements_v1.py +++ b/src/inference_endpoint/validation/schemas/requirements_v1.py @@ -13,7 +13,7 @@ AccuracyOperation, BindingMatch, ComparisonOperator, - DatasetCountOperator, + CountOperator, DerivedOperation, DrafterOperation, DurationBasis, @@ -25,7 +25,7 @@ SteadyOperation, WarmupOperation, ) -from ..types import AccuracyKind, FrozenModel, Identifier, OfflineMode +from ..types import AccuracyKind, FrozenModel, Identifier, OfflineMode, SampleUnit from ..vocabulary import CheckKind Number = Annotated[StrictInt | StrictFloat, Field(allow_inf_nan=False)] @@ -286,31 +286,28 @@ class CoverageRequirements(Requirements): target: Identifier -class DatasetCountRequirements(Requirements): +class CountRequirements(Requirements): catalog: Identifier - dataset: Identifier + dataset: Identifier | None = None + dataset_source: Identifier | None = None on_missing: Identifier | None = None - operator: DatasetCountOperator + on_unknown_or_null_threshold: Identifier | None = None + on_unsupported_sample_unit: Identifier | None = None + operator: CountOperator require_all: StrictBool | None = None - sample_unit: Identifier | None = None + sample_unit: SampleUnit | None = None source: Identifier | None = None sources: list[Identifier] | None = None + supported_sample_units: list[SampleUnit] | None = None + threshold: Identifier | None = None @model_validator(mode="after") def check_form(self) -> Self: self.require_exactly_one("source", "sources") + self.require_exactly_one("dataset", "dataset_source") return self -class DatasetMinimumRequirements(Requirements): - catalog: Identifier - on_unknown_or_null_threshold: Identifier | None = None - on_unsupported_sample_unit: Identifier | None = None - source: Identifier - supported_sample_units: list[Identifier] | None = None - threshold: Identifier - - class DerivedMetricRequirements(Requirements): constants: Identifier | None = None denominator: Identifier | None = None @@ -579,8 +576,7 @@ def check_form(self) -> Self: CheckKind.COMPARISON: ComparisonRequirements, CheckKind.CONSISTENCY: ConsistencyRequirements, CheckKind.COVERAGE: CoverageRequirements, - CheckKind.DATASET_COUNT: DatasetCountRequirements, - CheckKind.DATASET_MINIMUM: DatasetMinimumRequirements, + CheckKind.COUNT: CountRequirements, CheckKind.DERIVED_METRIC: DerivedMetricRequirements, CheckKind.DISCLOSURE: DisclosureRequirements, CheckKind.DRAFTER_BINDING: DrafterBindingRequirements, diff --git a/src/inference_endpoint/validation/vocabulary.py b/src/inference_endpoint/validation/vocabulary.py index 739e61589..8bde0ccb9 100644 --- a/src/inference_endpoint/validation/vocabulary.py +++ b/src/inference_endpoint/validation/vocabulary.py @@ -17,8 +17,7 @@ class CheckKind(StrEnum): COMPARISON = "comparison" CONSISTENCY = "consistency" COVERAGE = "coverage" - DATASET_COUNT = "dataset_count" - DATASET_MINIMUM = "dataset_minimum" + COUNT = "count" DERIVED_METRIC = "derived_metric" DISCLOSURE = "disclosure" DRAFTER_BINDING = "drafter_binding" diff --git a/tests/fixtures/validation/policy-fingerprints.json b/tests/fixtures/validation/policy-fingerprints.json index 33650e087..66dbbaaa7 100644 --- a/tests/fixtures/validation/policy-fingerprints.json +++ b/tests/fixtures/validation/policy-fingerprints.json @@ -45,8 +45,8 @@ "curve/agentic-osl-range": "e20feb64472b8faddcf31e08f3c18836c419ff239d942026bf275b0147597074", "point/submission-flags": "6fd56145219a078dd947d79bd1a5587e9c45da258085ea0e0145d3d75e7330a9", "point/swebench-template": "664ef45ae9124f181b8b1e947e25f29b6567ebb09c4ea192052c50678435dc15", - "point/agentic-trajectory-count": "5f22a6eea8421543f1a81e845f17be5e9f0ebbdd795d281e7889bc454430c18c", - "point/swebench-instance-count": "e9e031fe13262a9c7b60cf526def692e7d21085d84d0b10d5a9e2f575f3f42ea", + "point/trajectories-to-issue": "323a70bf8bca223b7f926afb53bfeacb519f59cc51949484b3643642bf2e79ab", + "point/swebench-instance-count": "84477a7f4dc7e0da91fabdf4e7ec73cbd886194eeadb3895136e4594980304fc", "point/endpoints-client-sha": "23b06e7e9a8dd4b39cf1cbd6809ec3f5bc2c80a49f460011b62f6dcb70ed8536", "point/approved-checkpoint": "fd35d43f5c4c9f4bb268a80c6d365226dfc53a25902fa98bf378add73e4766c3", "point/point-power": "6e75d380e17c6198e37ade0ac1ebd4598fc57c367a7994ff9543a75cb2106961", @@ -73,7 +73,7 @@ "point/steady-state-consistency": "9f751df4221317b44d13882c357c21b87e1ddbd1d62a88e3ee9e3dd89d40a5ae", "point/steady-state-basis": "c06a9eae5142326f1daab8db85b8a2bc8908f9d261ce3446d18b192b277f8177", "point/point-duration": "d5d109fd65f15b494af5554f82214e4d37adbe32b9407dc91a29ebba5037eafa", - "point/min-query-count": "b1200c70c4f53a8a1411f98babcb75441c9288eee32cea915f853157207e0de8", + "point/min-completed-samples": "2d88a3c7a939f0075e67cdbdcde7152a19aad7409df42073c33eb309bc497b87", "point/metric-consistency-duration": "e8dd3ddb3fd5d339147e4218ed178b20c620f0a8bcc5997ce70babdddf7b556b", "point/metric-consistency-accounting": "00600517efa73cb889b43fb4c31f7ae7b28de1bb8020da0a9c18e48369b08a03", "point/metric-consistency-output-tokens": "a9464734c55d01306cff0baf8b90aecce9c1052379726f11354cf1cf741461af", diff --git a/tests/unit/validation/test_counts.py b/tests/unit/validation/test_counts.py new file mode 100644 index 000000000..9c9cd9ded --- /dev/null +++ b/tests/unit/validation/test_counts.py @@ -0,0 +1,161 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Reported and configured counts share operators while retaining rule semantics.""" + +import json +import shutil +from dataclasses import replace + +import pytest +import yaml + +from inference_endpoint.validation import ( + bundled_policy_path, + load_policy, + validate_submission, +) + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def policy(): + return load_policy(bundled_policy_path()) + + +def check_count( + tmp_path, + policy, + rule_id, + *, + dataset="cnn_dailymail", + completed=13368, + trajectories=613, + declared_instances=200, + evaluated_instances=200, +): + point = tmp_path / "results" / "system" / "kimi-k3" / "r16" + point.mkdir(parents=True) + config = { + "concurrency": 16, + "dataset": dataset, + "runtime_settings": { + "load_pattern": "agentic_inference", + "runtime": {}, + "agentic_inference": {"num_trajectories_to_issue": trajectories}, + }, + "accuracy": { + "swe_bench": { + "extras": {"num_instances": declared_instances}, + "evaluated_instance_count": evaluated_instances, + } + }, + } + (point / "point.yaml").write_text(json.dumps(config)) + (point / "result_summary.json").write_text( + json.dumps( + { + "n_samples_completed": completed, + "duration_ns": 1e9, + } + ) + ) + selected = replace( + policy, checks=tuple(rule for rule in policy.checks if rule.id == rule_id) + ) + assert len(selected.checks) == 1 + return validate_submission(tmp_path, policy=selected) + + +@pytest.mark.parametrize( + ("completed", "passed"), [(13367, False), (13368, True), (20000, True)] +) +def test_min_completed_samples_accepts_extra_samples( + tmp_path, policy, completed, passed +): + report = check_count(tmp_path, policy, "min-completed-samples", completed=completed) + assert len(report.results) == 1 + assert report.passed is passed + + +@pytest.mark.parametrize( + ("trajectories", "passed"), + [(0, False), (612, False), (613, True), (614, False), (1226, True)], +) +def test_trajectories_to_issue_requires_positive_dataset_multiple( + tmp_path, policy, trajectories, passed +): + report = check_count( + tmp_path, policy, "trajectories-to-issue", trajectories=trajectories + ) + assert len(report.results) == 1 + assert report.passed is passed + + +@pytest.mark.parametrize( + ("declared", "evaluated", "passed"), + [ + (200, 200, True), + (199, 200, False), + (200, 199, False), + (400, 400, False), + (None, 200, False), + (200, None, False), + (True, 200, False), + ], +) +def test_swebench_instances_requires_both_exact_counts( + tmp_path, policy, declared, evaluated, passed +): + report = check_count( + tmp_path, + policy, + "swebench-instance-count", + declared_instances=declared, + evaluated_instances=evaluated, + ) + assert len(report.results) == 2 + assert report.passed is passed + + +@pytest.mark.parametrize("dataset", ["unknown", "agentic_combined", "swe_bench"]) +def test_min_completed_samples_skips_unknown_datasets_and_other_units( + tmp_path, policy, dataset +): + report = check_count( + tmp_path, policy, "min-completed-samples", dataset=dataset, completed=0 + ) + assert report.results == [] + + +@pytest.mark.parametrize( + "rule_id", ["trajectories-to-issue", "swebench-instance-count"] +) +def test_fixed_reference_count_missing_blocks_check(tmp_path, policy, rule_id): + policy = replace(policy, catalogs={**policy.catalogs, "datasets": {}}) + report = check_count(tmp_path, policy, rule_id) + assert report.errors + assert all(result.key == "blocked" for result in report.results) + + +@pytest.mark.parametrize( + "changes", + [ + {"dataset": "cnn_dailymail", "dataset_source": "point.dataset"}, + {"dataset": None}, + {"operator": "unknown"}, + { + "source": "result_summary.n_samples_completed", + "sources": ["result_summary.n_samples_issued"], + }, + ], +) +def test_count_requirements_reject_ambiguous_or_invalid_bindings(tmp_path, changes): + target = tmp_path / "2026-10-C1" + shutil.copytree(bundled_policy_path(), target) + path = target / "point_checks.yaml" + document = yaml.safe_load(path.read_text()) + document["checks"]["trajectories-to-issue"].update(changes) + path.write_text(yaml.safe_dump(document)) + with pytest.raises(ValueError, match="trajectories-to-issue"): + load_policy(target) diff --git a/tests/unit/validation/test_engine.py b/tests/unit/validation/test_engine.py index cb372e4bb..741bc4114 100644 --- a/tests/unit/validation/test_engine.py +++ b/tests/unit/validation/test_engine.py @@ -37,8 +37,8 @@ def policy_only(*ids): ) -def test_dataset_catalog_edit_changes_count_gate(): - policy = policy_only("min-query-count") +def test_min_completed_samples_uses_catalog_sample_count(): + policy = policy_only("min-completed-samples") original = execute_submission(FIXTURE, policy) catalog = { **policy.catalogs, @@ -47,7 +47,7 @@ def test_dataset_catalog_edit_changes_count_gate(): stricter = execute_submission(FIXTURE, replace(policy, catalogs=freeze(catalog))) assert original.passed assert not stricter.passed - assert all(f.rule == "min-query-count" for f in stricter.results) + assert all(f.rule == "min-completed-samples" for f in stricter.results) def test_duration_catalog_edit_changes_runtime_gate(): @@ -138,7 +138,7 @@ def test_disabled_cohort_override_prevents_execution(tmp_path): catalog["enrollment"]["overrides"] = [ { "id": "count-disabled", - "rule": "min-query-count", + "rule": "min-completed-samples", "reason": "Cohort count exemption", "applies_to": {"models": ["llama3_1-8b"]}, "enabled": False, @@ -148,7 +148,9 @@ def test_disabled_cohort_override_prevents_execution(tmp_path): policy = load_policy(policy_dir) policy = replace( policy, - checks=tuple(rule for rule in policy.checks if rule.id == "min-query-count"), + checks=tuple( + rule for rule in policy.checks if rule.id == "min-completed-samples" + ), ) assert validate_submission(FIXTURE, policy=policy).results == [] diff --git a/tests/unit/validation/test_loader.py b/tests/unit/validation/test_loader.py index dde40526d..396c0c434 100644 --- a/tests/unit/validation/test_loader.py +++ b/tests/unit/validation/test_loader.py @@ -126,9 +126,7 @@ def test_extra_files_wrong_scope_and_mixed_revisions_are_rejected(policy_dir): ) def test_unknown_dispatch_and_reference_values_are_rejected(policy_dir, change): point = policy_dir / PolicyFile.POINT.value - modify( - point, lambda data: data["checks"]["agentic-trajectory-count"].update(change) - ) + modify(point, lambda data: data["checks"]["trajectories-to-issue"].update(change)) with pytest.raises(ValueError): load_policy(policy_dir) @@ -154,7 +152,7 @@ def test_duplicate_ids_and_broken_catalog_bindings_are_rejected(policy_dir): def test_evidence_addresses_are_enum_backed(): policy = load_policy(bundled_policy_path()) - rule = next(rule for rule in policy.checks if rule.id == "agentic-trajectory-count") + rule = next(rule for rule in policy.checks if rule.id == "trajectories-to-issue") assert rule.scope is Scope.POINT assert ( EvidenceReference.POINT_RUNTIME_SETTINGS_AGENTIC_INFERENCE_NUM_TRAJECTORIES_TO_ISSUE diff --git a/tests/unit/validation/test_planner.py b/tests/unit/validation/test_planner.py index 6381b518f..cb6cd725c 100644 --- a/tests/unit/validation/test_planner.py +++ b/tests/unit/validation/test_planner.py @@ -50,12 +50,12 @@ def decisions(policy, context): def test_patterns_select_model_family_checks(policy): agentic = decisions(policy, point()) - assert agentic["agentic-trajectory-count"].decision is Decision.READY + assert agentic["trajectories-to-issue"].decision is Decision.READY assert agentic["approved-drafter"].decision is Decision.EXCLUDED single = decisions( policy, point(model_id="llama3_1-8b", load_pattern="concurrency") ) - assert single["agentic-trajectory-count"].decision is Decision.EXCLUDED + assert single["trajectories-to-issue"].decision is Decision.EXCLUDED assert single["point-config-valid"].decision is Decision.READY curve = Context.model_validate( { @@ -79,12 +79,12 @@ def test_exclusion_precedes_missing_dependencies(policy): ) assert checks["concurrency-in-range"].decision is Decision.EXCLUDED assert checks["load-pattern"].decision is Decision.EXCLUDED - assert checks["agentic-trajectory-count"].decision is Decision.EXCLUDED + assert checks["trajectories-to-issue"].decision is Decision.EXCLUDED def test_unknown_classification_blocks_without_hiding_unconditional_checks(policy): checks = decisions(policy, point(load_pattern=None, offline=None)) - assert checks["agentic-trajectory-count"].decision is Decision.BLOCKED + assert checks["trajectories-to-issue"].decision is Decision.BLOCKED assert checks["concurrency-in-range"].decision is Decision.BLOCKED assert checks["result-file-valid"].decision is Decision.READY @@ -101,8 +101,8 @@ def test_invalid_config_does_not_hide_summary_schema_checks(policy): ) assert checks["result-file-valid"].decision is Decision.READY assert checks["point-rules-skipped"].decision is Decision.READY - assert checks["agentic-trajectory-count"].decision is Decision.BLOCKED - assert checks["agentic-trajectory-count"].missing == {EvidenceKey.POINT_CONFIG} + assert checks["trajectories-to-issue"].decision is Decision.BLOCKED + assert checks["trajectories-to-issue"].missing == {EvidenceKey.POINT_CONFIG} def test_missing_seed_catalog_blocks_instead_of_deselecting(policy): @@ -174,7 +174,7 @@ def test_model_override_changes_only_selected_context(tmp_path): data = yaml.safe_load(path.read_text()) override = { "id": "kimi-count-exception", - "rule": "agentic-trajectory-count", + "rule": "trajectories-to-issue", "reason": "Fixture exception", "applies_to": {"models": ["kimi-k3"]}, "requirements": {"operator": "equal"}, @@ -182,11 +182,11 @@ def test_model_override_changes_only_selected_context(tmp_path): data["enrollment"]["overrides"] = [override] path.write_text(yaml.safe_dump(data, sort_keys=False)) policy = load_policy(directory) - check = decisions(policy, point())["agentic-trajectory-count"] + check = decisions(policy, point())["trajectories-to-issue"] assert check.override == "kimi-count-exception" assert check.rule.requirements["operator"] == "equal" other = decisions(policy, point(model_id="deepseek-v4_1-flash"))[ - "agentic-trajectory-count" + "trajectories-to-issue" ] assert other.override is None assert other.rule.requirements["operator"] == "positive_multiple" From 8b9538f00d0d074a6390f8bbbd004fde2ac8b440 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 03:35:59 -0700 Subject: [PATCH 12/18] docs(validation): rename implementation document to DESIGN.md --- AGENTS.md | 2 +- docs/validation/{implementation.md => DESIGN.md} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/validation/{implementation.md => DESIGN.md} (100%) diff --git a/AGENTS.md b/AGENTS.md index e40741d6a..774bdc084 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -100,7 +100,7 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint | **VideoGen** | `src/inference_endpoint/videogen/` | Adapter for video-generation endpoints (e.g. trtllm-serve `POST /v1/videos/generations`, used by MLPerf WAN2.2-T2V-A14B). Defaults to `response_format=video_path` (server saves video to shared storage and returns path) to avoid large byte payloads. Accuracy mode also runs on `video_path`: the adapter mirrors the path into `response_output` so the event log carries it to `VBenchScorer` (see `evaluation/scoring.py`), which scores videos via VBench from a sibling `uv` subproject at `examples/09_Wan22_VideoGen_Example/accuracy/` (vbench's `transformers==4.33.2` + `numpy<2` pins are incompatible with the parent env, so it runs out-of-process via `uv run --project`). Dataset is ingested via the generic JSONL loader. | | **SWE-bench** | `src/inference_endpoint/dataset_manager/predefined/swe_bench/`, `src/inference_endpoint/evaluation/swe_bench_scorer.py`, `src/inference_endpoint/evaluation/swebench_service/` | `SWEBench` predefined dataset (HuggingFace `princeton-nlp/SWE-bench_Verified` or `_Lite`; `ACCURACY_ONLY=True`). `SWEBenchScorer` sets `SKIP_ENDPOINT_PHASE=True` and bypasses the built-in accuracy phase entirely: it delegates agent execution and grading to the configured SWE-bench service via `accuracy_config.extras.swebench_service_url`. The service is an isolated `uv` subproject; its host owns Docker/runtime execution, artifacts, and credentials, while the benchmark client remains the report-producing entrypoint. | | **Compliance (submission checker)** | `src/inference_endpoint/compliance/checker.py`, `scripts/check_compliance.py` | Validates a completed run's report directory against a registered ruleset. `check_submission(report_dir, ruleset, model)` reads the resolved `config.yaml` plus scorer output (`accuracy/accuracy_results.json` for accuracy, `scores.json` for the agentic perf run) and runs config-lock (deterministic + single-stream), the accuracy gate (`score >= factor x reference`, factor 0.97 for Edge-Agentic), and run validity (0 dropped turns). Server-side launch flags (`--reasoning off`, `--ctx-size`) aren't in client artifacts, so they're surfaced as manual attestations. CLI: `scripts/check_compliance.py REPORT_DIR` (exit 0 = pass). | -| **Submission validation** | `src/inference_endpoint/validation/` | Artifact validation, five-file cohort policies, typed evidence schemas with supplied-field metadata, grouped artifact indexes and derived state, check planning and policy-driven power calculations. See `docs/validation/implementation.md`. | +| **Submission validation** | `src/inference_endpoint/validation/` | Artifact validation, five-file cohort policies, typed evidence schemas with supplied-field metadata, grouped artifact indexes and derived state, check planning and policy-driven power calculations. See `docs/validation/DESIGN.md`. | | **Compliance (audit tests)** | `src/inference_endpoint/compliance/`, `commands/audit.py` | MLPerf compliance audits. `AuditTest` protocol + `AuditRunSpec`/`AuditRunArtifacts` + registry (`compliance/__init__.py`); `OutputCachingAudit` (`compliance/audit_test/output_caching_test.py`, which also owns the QPS-specific `AuditRunStats`) implements MLPerf **TEST04** output-caching detection — reference phase (distinct samples) vs. fixed-sample audit phase, comparing QPS against `threshold`. `commands/audit.py:run_audit` runs phases via `AuditTest.plan_runs`/`validate`, writing `audit_result.json`/`verify_.txt` atomically via `compliance/result.py`. Enabled by the `audit:` YAML block; `cli._run` runs it after the main benchmark (upstream MLPerf order: perf run, then TEST04), or standalone with `audit.only: true`. Perf-only by default (a phase may opt into accuracy via `AuditRunSpec.test_mode`, but this is unused today). | ### Hot-Path Architecture diff --git a/docs/validation/implementation.md b/docs/validation/DESIGN.md similarity index 100% rename from docs/validation/implementation.md rename to docs/validation/DESIGN.md From 4f9e35b779d690ee2fae749750487995e4063198 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 05:26:18 -0700 Subject: [PATCH 13/18] test(validation): use neutral names for synthetic submission systems --- tests/fixtures/validation/README.md | 25 +++++++++++++++++++ .../r16/accuracy_results.json | 0 .../Llama-3_1-8B-Instruct/r16/config.yaml | 0 .../Llama-3_1-8B-Instruct/r16/point.yaml | 0 .../r16/result_summary.json | 0 .../r16/system_desc.json | 2 +- .../r38/accuracy_results.json | 0 .../Llama-3_1-8B-Instruct/r38/config.yaml | 0 .../Llama-3_1-8B-Instruct/r38/point.yaml | 0 .../r38/result_summary.json | 0 .../r38/system_desc.json | 2 +- .../r88/accuracy_results.json | 0 .../Llama-3_1-8B-Instruct/r88/config.yaml | 0 .../Llama-3_1-8B-Instruct/r88/point.yaml | 0 .../r88/result_summary.json | 0 .../r88/system_desc.json | 2 +- .../system_power.json | 2 +- .../gpt-oss-120b/r1024/accuracy_results.json | 0 .../gpt-oss-120b/r1024/config.yaml | 0 .../gpt-oss-120b/r1024/point.yaml | 0 .../gpt-oss-120b/r1024/result_summary.json | 0 .../gpt-oss-120b/r1024/system_desc.json | 2 +- .../gpt-oss-120b/r128/accuracy_results.json | 0 .../gpt-oss-120b/r128/config.yaml | 0 .../gpt-oss-120b/r128/point.yaml | 0 .../gpt-oss-120b/r128/result_summary.json | 0 .../gpt-oss-120b/r128/system_desc.json | 2 +- .../gpt-oss-120b/r16/accuracy_results.json | 0 .../gpt-oss-120b/r16/config.yaml | 0 .../gpt-oss-120b/r16/point.yaml | 0 .../gpt-oss-120b/r16/result_summary.json | 0 .../gpt-oss-120b/r16/system_desc.json | 2 +- .../gpt-oss-120b/r2048/accuracy_results.json | 0 .../gpt-oss-120b/r2048/config.yaml | 0 .../gpt-oss-120b/r2048/point.yaml | 0 .../gpt-oss-120b/r2048/result_summary.json | 0 .../gpt-oss-120b/r2048/system_desc.json | 2 +- .../gpt-oss-120b/r4/accuracy_results.json | 0 .../gpt-oss-120b/r4/config.yaml | 0 .../gpt-oss-120b/r4/point.yaml | 0 .../gpt-oss-120b/r4/result_summary.json | 0 .../gpt-oss-120b/r4/system_desc.json | 2 +- .../gpt-oss-120b/r512/accuracy_results.json | 0 .../gpt-oss-120b/r512/config.yaml | 0 .../gpt-oss-120b/r512/point.yaml | 0 .../gpt-oss-120b/r512/result_summary.json | 0 .../gpt-oss-120b/r512/system_desc.json | 2 +- .../gpt-oss-120b/r64/accuracy_results.json | 0 .../gpt-oss-120b/r64/config.yaml | 0 .../gpt-oss-120b/r64/point.yaml | 0 .../gpt-oss-120b/r64/result_summary.json | 0 .../gpt-oss-120b/r64/system_desc.json | 2 +- .../system_power.json | 2 +- .../gpt-oss-120b/r1024/accuracy_results.json | 0 .../gpt-oss-120b/r1024/config.yaml | 0 .../gpt-oss-120b/r1024/point.yaml | 0 .../gpt-oss-120b/r1024/result_summary.json | 0 .../gpt-oss-120b/r1024/system_desc.json | 2 +- .../gpt-oss-120b/r128/accuracy_results.json | 0 .../gpt-oss-120b/r128/config.yaml | 0 .../gpt-oss-120b/r128/point.yaml | 0 .../gpt-oss-120b/r128/result_summary.json | 0 .../gpt-oss-120b/r128/system_desc.json | 2 +- .../gpt-oss-120b/r16/accuracy_results.json | 0 .../gpt-oss-120b/r16/config.yaml | 0 .../gpt-oss-120b/r16/point.yaml | 0 .../gpt-oss-120b/r16/result_summary.json | 0 .../gpt-oss-120b/r16/system_desc.json | 2 +- .../gpt-oss-120b/r2048/accuracy_results.json | 0 .../gpt-oss-120b/r2048/config.yaml | 0 .../gpt-oss-120b/r2048/point.yaml | 0 .../gpt-oss-120b/r2048/result_summary.json | 0 .../gpt-oss-120b/r2048/system_desc.json | 2 +- .../gpt-oss-120b/r4/accuracy_results.json | 0 .../gpt-oss-120b/r4/config.yaml | 0 .../gpt-oss-120b/r4/point.yaml | 0 .../gpt-oss-120b/r4/result_summary.json | 0 .../gpt-oss-120b/r4/system_desc.json | 2 +- .../gpt-oss-120b/r512/accuracy_results.json | 0 .../gpt-oss-120b/r512/config.yaml | 0 .../gpt-oss-120b/r512/point.yaml | 0 .../gpt-oss-120b/r512/result_summary.json | 0 .../gpt-oss-120b/r512/system_desc.json | 2 +- .../gpt-oss-120b/r64/accuracy_results.json | 0 .../gpt-oss-120b/r64/config.yaml | 0 .../gpt-oss-120b/r64/point.yaml | 0 .../gpt-oss-120b/r64/result_summary.json | 0 .../gpt-oss-120b/r64/system_desc.json | 2 +- .../system_power.json | 2 +- .../r128/accuracy_results.json | 0 .../qwen3-coder-480b/r128/config.yaml | 0 .../qwen3-coder-480b/r128/point.yaml | 0 .../qwen3-coder-480b/r128/result_summary.json | 0 .../qwen3-coder-480b/r128/system_desc.json | 2 +- .../r16/accuracy_results.json | 0 .../qwen3-coder-480b/r16/config.yaml | 0 .../qwen3-coder-480b/r16/point.yaml | 0 .../qwen3-coder-480b/r16/result_summary.json | 0 .../qwen3-coder-480b/r16/system_desc.json | 2 +- .../r256/accuracy_results.json | 0 .../qwen3-coder-480b/r256/config.yaml | 0 .../qwen3-coder-480b/r256/point.yaml | 0 .../qwen3-coder-480b/r256/result_summary.json | 0 .../qwen3-coder-480b/r256/system_desc.json | 2 +- .../r32/accuracy_results.json | 0 .../qwen3-coder-480b/r32/config.yaml | 0 .../qwen3-coder-480b/r32/point.yaml | 0 .../qwen3-coder-480b/r32/result_summary.json | 0 .../qwen3-coder-480b/r32/system_desc.json | 2 +- .../r512/accuracy_results.json | 0 .../qwen3-coder-480b/r512/config.yaml | 0 .../qwen3-coder-480b/r512/point.yaml | 0 .../qwen3-coder-480b/r512/result_summary.json | 0 .../qwen3-coder-480b/r512/system_desc.json | 2 +- .../r64/accuracy_results.json | 0 .../qwen3-coder-480b/r64/config.yaml | 0 .../qwen3-coder-480b/r64/point.yaml | 0 .../qwen3-coder-480b/r64/result_summary.json | 0 .../qwen3-coder-480b/r64/system_desc.json | 2 +- .../qwen3-coder-480b/r8/accuracy_results.json | 0 .../qwen3-coder-480b/r8/config.yaml | 0 .../qwen3-coder-480b/r8/point.yaml | 0 .../qwen3-coder-480b/r8/result_summary.json | 0 .../qwen3-coder-480b/r8/system_desc.json | 2 +- .../system_power.json | 2 +- .../r1024/accuracy_results.json | 0 .../qwen3-coder-480b/r1024/config.yaml | 0 .../qwen3-coder-480b/r1024/point.yaml | 0 .../r1024/result_summary.json | 0 .../qwen3-coder-480b/r1024/system_desc.json | 2 +- .../r128/accuracy_results.json | 0 .../qwen3-coder-480b/r128/config.yaml | 0 .../qwen3-coder-480b/r128/point.yaml | 0 .../qwen3-coder-480b/r128/result_summary.json | 0 .../qwen3-coder-480b/r128/system_desc.json | 2 +- .../r16/accuracy_results.json | 0 .../qwen3-coder-480b/r16/config.yaml | 0 .../qwen3-coder-480b/r16/point.yaml | 0 .../qwen3-coder-480b/r16/result_summary.json | 0 .../qwen3-coder-480b/r16/system_desc.json | 2 +- .../r256/accuracy_results.json | 0 .../qwen3-coder-480b/r256/config.yaml | 0 .../qwen3-coder-480b/r256/point.yaml | 0 .../qwen3-coder-480b/r256/result_summary.json | 0 .../qwen3-coder-480b/r256/system_desc.json | 2 +- .../r32/accuracy_results.json | 0 .../qwen3-coder-480b/r32/config.yaml | 0 .../qwen3-coder-480b/r32/point.yaml | 0 .../qwen3-coder-480b/r32/result_summary.json | 0 .../qwen3-coder-480b/r32/system_desc.json | 2 +- .../r512/accuracy_results.json | 0 .../qwen3-coder-480b/r512/config.yaml | 0 .../qwen3-coder-480b/r512/point.yaml | 0 .../qwen3-coder-480b/r512/result_summary.json | 0 .../qwen3-coder-480b/r512/system_desc.json | 2 +- .../r64/accuracy_results.json | 0 .../qwen3-coder-480b/r64/config.yaml | 0 .../qwen3-coder-480b/r64/point.yaml | 0 .../qwen3-coder-480b/r64/result_summary.json | 0 .../qwen3-coder-480b/r64/system_desc.json | 2 +- .../qwen3-coder-480b/r8/accuracy_results.json | 0 .../qwen3-coder-480b/r8/config.yaml | 0 .../qwen3-coder-480b/r8/point.yaml | 0 .../qwen3-coder-480b/r8/result_summary.json | 0 .../qwen3-coder-480b/r8/system_desc.json | 2 +- .../system_power.json | 2 +- .../llama3-8b/r1/accuracy_results.json | 0 .../llama3-8b/r1/config.yaml | 0 .../llama3-8b/r1/point.yaml | 0 .../llama3-8b/r1/result_summary.json | 0 .../llama3-8b/r1/system_desc.json | 2 +- .../llama3-8b/r1024/accuracy_results.json | 0 .../llama3-8b/r1024/config.yaml | 0 .../llama3-8b/r1024/point.yaml | 0 .../llama3-8b/r1024/result_summary.json | 0 .../llama3-8b/r1024/system_desc.json | 2 +- .../llama3-8b/r128/accuracy_results.json | 0 .../llama3-8b/r128/config.yaml | 0 .../llama3-8b/r128/point.yaml | 0 .../llama3-8b/r128/result_summary.json | 0 .../llama3-8b/r128/system_desc.json | 2 +- .../llama3-8b/r16/accuracy_results.json | 0 .../llama3-8b/r16/config.yaml | 0 .../llama3-8b/r16/point.yaml | 0 .../llama3-8b/r16/result_summary.json | 0 .../llama3-8b/r16/system_desc.json | 2 +- .../llama3-8b/r2/accuracy_results.json | 0 .../llama3-8b/r2/config.yaml | 0 .../llama3-8b/r2/point.yaml | 0 .../llama3-8b/r2/result_summary.json | 0 .../llama3-8b/r2/system_desc.json | 2 +- .../llama3-8b/r256/accuracy_results.json | 0 .../llama3-8b/r256/config.yaml | 0 .../llama3-8b/r256/point.yaml | 0 .../llama3-8b/r256/result_summary.json | 0 .../llama3-8b/r256/system_desc.json | 2 +- .../llama3-8b/r32/accuracy_results.json | 0 .../llama3-8b/r32/config.yaml | 0 .../llama3-8b/r32/point.yaml | 0 .../llama3-8b/r32/result_summary.json | 0 .../llama3-8b/r32/system_desc.json | 2 +- .../llama3-8b/r4/accuracy_results.json | 0 .../llama3-8b/r4/config.yaml | 0 .../llama3-8b/r4/point.yaml | 0 .../llama3-8b/r4/result_summary.json | 0 .../llama3-8b/r4/system_desc.json | 2 +- .../llama3-8b/r512/accuracy_results.json | 0 .../llama3-8b/r512/config.yaml | 0 .../llama3-8b/r512/point.yaml | 0 .../llama3-8b/r512/result_summary.json | 0 .../llama3-8b/r512/system_desc.json | 2 +- .../llama3-8b/r64/accuracy_results.json | 0 .../llama3-8b/r64/config.yaml | 0 .../llama3-8b/r64/point.yaml | 0 .../llama3-8b/r64/result_summary.json | 0 .../llama3-8b/r64/system_desc.json | 2 +- .../llama3-8b/r8/accuracy_results.json | 0 .../llama3-8b/r8/config.yaml | 0 .../llama3-8b/r8/point.yaml | 0 .../llama3-8b/r8/result_summary.json | 0 .../llama3-8b/r8/system_desc.json | 2 +- .../system_power.json | 2 +- .../llama3-8b/r1/accuracy_results.json | 0 .../llama3-8b/r1/config.yaml | 0 .../llama3-8b/r1/point.yaml | 0 .../llama3-8b/r1/result_summary.json | 0 .../llama3-8b/r1/system_desc.json | 2 +- .../llama3-8b/r1024/accuracy_results.json | 0 .../llama3-8b/r1024/config.yaml | 0 .../llama3-8b/r1024/point.yaml | 0 .../llama3-8b/r1024/result_summary.json | 0 .../llama3-8b/r1024/system_desc.json | 2 +- .../llama3-8b/r128/accuracy_results.json | 0 .../llama3-8b/r128/config.yaml | 0 .../llama3-8b/r128/point.yaml | 0 .../llama3-8b/r128/result_summary.json | 0 .../llama3-8b/r128/system_desc.json | 2 +- .../llama3-8b/r16/accuracy_results.json | 0 .../llama3-8b/r16/config.yaml | 0 .../llama3-8b/r16/point.yaml | 0 .../llama3-8b/r16/result_summary.json | 0 .../llama3-8b/r16/system_desc.json | 2 +- .../llama3-8b/r2/accuracy_results.json | 0 .../llama3-8b/r2/config.yaml | 0 .../llama3-8b/r2/point.yaml | 0 .../llama3-8b/r2/result_summary.json | 0 .../llama3-8b/r2/system_desc.json | 2 +- .../llama3-8b/r256/accuracy_results.json | 0 .../llama3-8b/r256/config.yaml | 0 .../llama3-8b/r256/point.yaml | 0 .../llama3-8b/r256/result_summary.json | 0 .../llama3-8b/r256/system_desc.json | 2 +- .../llama3-8b/r32/accuracy_results.json | 0 .../llama3-8b/r32/config.yaml | 0 .../llama3-8b/r32/point.yaml | 0 .../llama3-8b/r32/result_summary.json | 0 .../llama3-8b/r32/system_desc.json | 2 +- .../llama3-8b/r4/accuracy_results.json | 0 .../llama3-8b/r4/config.yaml | 0 .../llama3-8b/r4/point.yaml | 0 .../llama3-8b/r4/result_summary.json | 0 .../llama3-8b/r4/system_desc.json | 2 +- .../llama3-8b/r512/accuracy_results.json | 0 .../llama3-8b/r512/config.yaml | 0 .../llama3-8b/r512/point.yaml | 0 .../llama3-8b/r512/result_summary.json | 0 .../llama3-8b/r512/system_desc.json | 2 +- .../llama3-8b/r64/accuracy_results.json | 0 .../llama3-8b/r64/config.yaml | 0 .../llama3-8b/r64/point.yaml | 0 .../llama3-8b/r64/result_summary.json | 0 .../llama3-8b/r64/system_desc.json | 2 +- .../llama3-8b/r8/accuracy_results.json | 0 .../llama3-8b/r8/config.yaml | 0 .../llama3-8b/r8/point.yaml | 0 .../llama3-8b/r8/result_summary.json | 0 .../llama3-8b/r8/system_desc.json | 2 +- .../system_power.json | 2 +- .../llama3-70b/r1024/accuracy_results.json | 0 .../llama3-70b/r1024/config.yaml | 0 .../llama3-70b/r1024/point.yaml | 0 .../llama3-70b/r1024/result_summary.json | 0 .../llama3-70b/r1024/system_desc.json | 2 +- .../llama3-70b/r128/accuracy_results.json | 0 .../llama3-70b/r128/config.yaml | 0 .../llama3-70b/r128/point.yaml | 0 .../llama3-70b/r128/result_summary.json | 0 .../llama3-70b/r128/system_desc.json | 2 +- .../llama3-70b/r1536/accuracy_results.json | 0 .../llama3-70b/r1536/config.yaml | 0 .../llama3-70b/r1536/point.yaml | 0 .../llama3-70b/r1536/result_summary.json | 0 .../llama3-70b/r1536/system_desc.json | 2 +- .../llama3-70b/r192/accuracy_results.json | 0 .../llama3-70b/r192/config.yaml | 0 .../llama3-70b/r192/point.yaml | 0 .../llama3-70b/r192/result_summary.json | 0 .../llama3-70b/r192/system_desc.json | 2 +- .../llama3-70b/r2048/accuracy_results.json | 0 .../llama3-70b/r2048/config.yaml | 0 .../llama3-70b/r2048/point.yaml | 0 .../llama3-70b/r2048/result_summary.json | 0 .../llama3-70b/r2048/system_desc.json | 2 +- .../llama3-70b/r256/accuracy_results.json | 0 .../llama3-70b/r256/config.yaml | 0 .../llama3-70b/r256/point.yaml | 0 .../llama3-70b/r256/result_summary.json | 0 .../llama3-70b/r256/system_desc.json | 2 +- .../llama3-70b/r384/accuracy_results.json | 0 .../llama3-70b/r384/config.yaml | 0 .../llama3-70b/r384/point.yaml | 0 .../llama3-70b/r384/result_summary.json | 0 .../llama3-70b/r384/system_desc.json | 2 +- .../llama3-70b/r512/accuracy_results.json | 0 .../llama3-70b/r512/config.yaml | 0 .../llama3-70b/r512/point.yaml | 0 .../llama3-70b/r512/result_summary.json | 0 .../llama3-70b/r512/system_desc.json | 2 +- .../llama3-70b/r64/accuracy_results.json | 0 .../llama3-70b/r64/config.yaml | 0 .../llama3-70b/r64/point.yaml | 0 .../llama3-70b/r64/result_summary.json | 0 .../llama3-70b/r64/system_desc.json | 2 +- .../llama3-70b/r768/accuracy_results.json | 0 .../llama3-70b/r768/config.yaml | 0 .../llama3-70b/r768/point.yaml | 0 .../llama3-70b/r768/result_summary.json | 0 .../llama3-70b/r768/system_desc.json | 2 +- .../system_power.json | 2 +- .../llama3-70b/r1024/accuracy_results.json | 0 .../llama3-70b/r1024/config.yaml | 0 .../llama3-70b/r1024/point.yaml | 0 .../llama3-70b/r1024/result_summary.json | 0 .../llama3-70b/r1024/system_desc.json | 2 +- .../llama3-70b/r128/accuracy_results.json | 0 .../llama3-70b/r128/config.yaml | 0 .../llama3-70b/r128/point.yaml | 0 .../llama3-70b/r128/result_summary.json | 0 .../llama3-70b/r128/system_desc.json | 2 +- .../llama3-70b/r1536/accuracy_results.json | 0 .../llama3-70b/r1536/config.yaml | 0 .../llama3-70b/r1536/point.yaml | 0 .../llama3-70b/r1536/result_summary.json | 0 .../llama3-70b/r1536/system_desc.json | 2 +- .../llama3-70b/r192/accuracy_results.json | 0 .../llama3-70b/r192/config.yaml | 0 .../llama3-70b/r192/point.yaml | 0 .../llama3-70b/r192/result_summary.json | 0 .../llama3-70b/r192/system_desc.json | 2 +- .../llama3-70b/r2048/accuracy_results.json | 0 .../llama3-70b/r2048/config.yaml | 0 .../llama3-70b/r2048/point.yaml | 0 .../llama3-70b/r2048/result_summary.json | 0 .../llama3-70b/r2048/system_desc.json | 2 +- .../llama3-70b/r256/accuracy_results.json | 0 .../llama3-70b/r256/config.yaml | 0 .../llama3-70b/r256/point.yaml | 0 .../llama3-70b/r256/result_summary.json | 0 .../llama3-70b/r256/system_desc.json | 2 +- .../llama3-70b/r384/accuracy_results.json | 0 .../llama3-70b/r384/config.yaml | 0 .../llama3-70b/r384/point.yaml | 0 .../llama3-70b/r384/result_summary.json | 0 .../llama3-70b/r384/system_desc.json | 2 +- .../llama3-70b/r512/accuracy_results.json | 0 .../llama3-70b/r512/config.yaml | 0 .../llama3-70b/r512/point.yaml | 0 .../llama3-70b/r512/result_summary.json | 0 .../llama3-70b/r512/system_desc.json | 2 +- .../llama3-70b/r64/accuracy_results.json | 0 .../llama3-70b/r64/config.yaml | 0 .../llama3-70b/r64/point.yaml | 0 .../llama3-70b/r64/result_summary.json | 0 .../llama3-70b/r64/system_desc.json | 2 +- .../llama3-70b/r768/accuracy_results.json | 0 .../llama3-70b/r768/config.yaml | 0 .../llama3-70b/r768/point.yaml | 0 .../llama3-70b/r768/result_summary.json | 0 .../llama3-70b/r768/system_desc.json | 2 +- .../system_power.json | 2 +- .../deepseek-r1/r1/accuracy_results.json | 0 .../deepseek-r1/r1/config.yaml | 0 .../deepseek-r1/r1/point.yaml | 0 .../deepseek-r1/r1/result_summary.json | 0 .../deepseek-r1/r1/system_desc.json | 2 +- .../deepseek-r1/r128/accuracy_results.json | 0 .../deepseek-r1/r128/config.yaml | 0 .../deepseek-r1/r128/point.yaml | 0 .../deepseek-r1/r128/result_summary.json | 0 .../deepseek-r1/r128/system_desc.json | 2 +- .../deepseek-r1/r16/accuracy_results.json | 0 .../deepseek-r1/r16/config.yaml | 0 .../deepseek-r1/r16/point.yaml | 0 .../deepseek-r1/r16/result_summary.json | 0 .../deepseek-r1/r16/system_desc.json | 2 +- .../deepseek-r1/r2/accuracy_results.json | 0 .../deepseek-r1/r2/config.yaml | 0 .../deepseek-r1/r2/point.yaml | 0 .../deepseek-r1/r2/result_summary.json | 0 .../deepseek-r1/r2/system_desc.json | 2 +- .../deepseek-r1/r256/accuracy_results.json | 0 .../deepseek-r1/r256/config.yaml | 0 .../deepseek-r1/r256/point.yaml | 0 .../deepseek-r1/r256/result_summary.json | 0 .../deepseek-r1/r256/system_desc.json | 2 +- .../deepseek-r1/r32/accuracy_results.json | 0 .../deepseek-r1/r32/config.yaml | 0 .../deepseek-r1/r32/point.yaml | 0 .../deepseek-r1/r32/result_summary.json | 0 .../deepseek-r1/r32/system_desc.json | 2 +- .../deepseek-r1/r4/accuracy_results.json | 0 .../deepseek-r1/r4/config.yaml | 0 .../deepseek-r1/r4/point.yaml | 0 .../deepseek-r1/r4/result_summary.json | 0 .../deepseek-r1/r4/system_desc.json | 2 +- .../deepseek-r1/r512/accuracy_results.json | 0 .../deepseek-r1/r512/config.yaml | 0 .../deepseek-r1/r512/point.yaml | 0 .../deepseek-r1/r512/result_summary.json | 0 .../deepseek-r1/r512/system_desc.json | 2 +- .../deepseek-r1/r64/accuracy_results.json | 0 .../deepseek-r1/r64/config.yaml | 0 .../deepseek-r1/r64/point.yaml | 0 .../deepseek-r1/r64/result_summary.json | 0 .../deepseek-r1/r64/system_desc.json | 2 +- .../deepseek-r1/r8/accuracy_results.json | 0 .../deepseek-r1/r8/config.yaml | 0 .../deepseek-r1/r8/point.yaml | 0 .../deepseek-r1/r8/result_summary.json | 0 .../deepseek-r1/r8/system_desc.json | 2 +- .../system_power.json | 2 +- .../deepseek-r1/r1024/accuracy_results.json | 0 .../deepseek-r1/r1024/config.yaml | 0 .../deepseek-r1/r1024/point.yaml | 0 .../deepseek-r1/r1024/result_summary.json | 0 .../deepseek-r1/r1024/system_desc.json | 2 +- .../deepseek-r1/r128/accuracy_results.json | 0 .../deepseek-r1/r128/config.yaml | 0 .../deepseek-r1/r128/point.yaml | 0 .../deepseek-r1/r128/result_summary.json | 0 .../deepseek-r1/r128/system_desc.json | 2 +- .../deepseek-r1/r16384/accuracy_results.json | 0 .../deepseek-r1/r16384/config.yaml | 0 .../deepseek-r1/r16384/point.yaml | 0 .../deepseek-r1/r16384/result_summary.json | 0 .../deepseek-r1/r16384/system_desc.json | 2 +- .../deepseek-r1/r2048/accuracy_results.json | 0 .../deepseek-r1/r2048/config.yaml | 0 .../deepseek-r1/r2048/point.yaml | 0 .../deepseek-r1/r2048/result_summary.json | 0 .../deepseek-r1/r2048/system_desc.json | 2 +- .../deepseek-r1/r256/accuracy_results.json | 0 .../deepseek-r1/r256/config.yaml | 0 .../deepseek-r1/r256/point.yaml | 0 .../deepseek-r1/r256/result_summary.json | 0 .../deepseek-r1/r256/system_desc.json | 2 +- .../deepseek-r1/r32/accuracy_results.json | 0 .../deepseek-r1/r32/config.yaml | 0 .../deepseek-r1/r32/point.yaml | 0 .../deepseek-r1/r32/result_summary.json | 0 .../deepseek-r1/r32/system_desc.json | 2 +- .../deepseek-r1/r4096/accuracy_results.json | 0 .../deepseek-r1/r4096/config.yaml | 0 .../deepseek-r1/r4096/point.yaml | 0 .../deepseek-r1/r4096/result_summary.json | 0 .../deepseek-r1/r4096/system_desc.json | 2 +- .../deepseek-r1/r512/accuracy_results.json | 0 .../deepseek-r1/r512/config.yaml | 0 .../deepseek-r1/r512/point.yaml | 0 .../deepseek-r1/r512/result_summary.json | 0 .../deepseek-r1/r512/system_desc.json | 2 +- .../deepseek-r1/r64/accuracy_results.json | 0 .../deepseek-r1/r64/config.yaml | 0 .../deepseek-r1/r64/point.yaml | 0 .../deepseek-r1/r64/result_summary.json | 0 .../deepseek-r1/r64/system_desc.json | 2 +- .../deepseek-r1/r8192/accuracy_results.json | 0 .../deepseek-r1/r8192/config.yaml | 0 .../deepseek-r1/r8192/point.yaml | 0 .../deepseek-r1/r8192/result_summary.json | 0 .../deepseek-r1/r8192/system_desc.json | 2 +- .../system_power.json | 2 +- .../llama3_1-8b/r1000/accuracy_results.json | 0 .../llama3_1-8b/r1000/config.yaml | 0 .../llama3_1-8b/r1000/point.yaml | 0 .../llama3_1-8b/r1000/result_summary.json | 0 .../llama3_1-8b/r1000/system_desc.json | 0 .../llama3_1-8b/r16/accuracy_results.json | 0 .../llama3_1-8b/r16/config.yaml | 0 .../llama3_1-8b/r16/point.yaml | 0 .../llama3_1-8b/r16/result_summary.json | 0 .../llama3_1-8b/r16/system_desc.json | 0 .../llama3_1-8b/r20/accuracy_results.json | 0 .../llama3_1-8b/r20/config.yaml | 0 .../llama3_1-8b/r20/point.yaml | 0 .../llama3_1-8b/r20/result_summary.json | 0 .../llama3_1-8b/r20/system_desc.json | 0 .../llama3_1-8b/r256/accuracy_results.json | 0 .../llama3_1-8b/r256/config.yaml | 0 .../llama3_1-8b/r256/point.yaml | 0 .../llama3_1-8b/r256/result_summary.json | 0 .../llama3_1-8b/r256/system_desc.json | 0 .../llama3_1-8b/r512/accuracy_results.json | 0 .../llama3_1-8b/r512/config.yaml | 0 .../llama3_1-8b/r512/point.yaml | 0 .../llama3_1-8b/r512/result_summary.json | 0 .../llama3_1-8b/r512/system_desc.json | 0 .../llama3_1-8b/r768/accuracy_results.json | 0 .../llama3_1-8b/r768/config.yaml | 0 .../llama3_1-8b/r768/point.yaml | 0 .../llama3_1-8b/r768/result_summary.json | 0 .../llama3_1-8b/r768/system_desc.json | 0 .../llama3_1-8b/r88/accuracy_results.json | 0 .../llama3_1-8b/r88/config.yaml | 0 .../llama3_1-8b/r88/point.yaml | 0 .../llama3_1-8b/r88/result_summary.json | 0 .../llama3_1-8b/r88/system_desc.json | 0 .../system_power.json | 2 +- .../unit/validation/test_policy_contracts.py | 2 +- 519 files changed, 132 insertions(+), 107 deletions(-) create mode 100644 tests/fixtures/validation/README.md rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r16/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r16/config.yaml (100%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r16/point.yaml (100%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r16/result_summary.json (100%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r16/system_desc.json (98%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r38/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r38/config.yaml (100%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r38/point.yaml (100%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r38/result_summary.json (100%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r38/system_desc.json (98%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r88/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r88/config.yaml (100%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r88/point.yaml (100%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r88/result_summary.json (100%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/Llama-3_1-8B-Instruct/r88/system_desc.json (98%) rename tests/fixtures/validation/submissions/invalid_submission/results/{bad_system => system_l}/system_power.json (96%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r1024/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r1024/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r1024/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r1024/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r1024/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r128/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r128/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r128/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r128/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r128/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r16/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r16/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r16/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r16/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r16/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r2048/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r2048/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r2048/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r2048/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r2048/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r4/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r4/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r4/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r4/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r4/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r512/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r512/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r512/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r512/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r512/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r64/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r64/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r64/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r64/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/gpt-oss-120b/r64/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_a/results/{sys_mi355x_8gpu => system_a}/system_power.json (96%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r1024/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r1024/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r1024/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r1024/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r1024/system_desc.json (97%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r128/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r128/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r128/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r128/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r128/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r16/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r16/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r16/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r16/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r16/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r2048/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r2048/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r2048/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r2048/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r2048/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r4/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r4/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r4/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r4/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r4/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r512/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r512/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r512/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r512/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r512/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r64/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r64/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r64/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r64/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/gpt-oss-120b/r64/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_b/results/{sys_mi355x_16gpu => system_b}/system_power.json (96%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r128/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r128/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r128/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r128/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r128/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r16/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r16/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r16/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r16/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r16/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r256/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r256/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r256/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r256/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r256/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r32/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r32/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r32/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r32/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r32/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r512/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r512/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r512/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r512/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r512/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r64/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r64/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r64/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r64/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r64/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r8/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r8/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r8/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r8/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/qwen3-coder-480b/r8/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_c/results/{sys_tpu_4chip => system_c}/system_power.json (96%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r1024/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r1024/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r1024/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r1024/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r1024/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r128/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r128/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r128/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r128/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r128/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r16/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r16/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r16/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r16/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r16/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r256/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r256/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r256/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r256/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r256/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r32/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r32/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r32/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r32/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r32/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r512/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r512/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r512/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r512/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r512/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r64/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r64/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r64/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r64/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r64/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r8/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r8/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r8/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r8/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/qwen3-coder-480b/r8/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_d/results/{sys_tpu_8chip => system_d}/system_power.json (96%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r1/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r1/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r1/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r1/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r1/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r1024/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r1024/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r1024/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r1024/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r1024/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r128/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r128/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r128/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r128/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r128/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r16/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r16/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r16/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r16/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r16/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r2/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r2/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r2/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r2/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r2/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r256/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r256/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r256/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r256/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r256/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r32/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r32/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r32/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r32/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r32/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r4/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r4/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r4/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r4/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r4/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r512/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r512/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r512/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r512/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r512/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r64/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r64/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r64/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r64/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r64/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r8/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r8/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r8/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r8/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/llama3-8b/r8/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_e/results/{sys_gaudi_dp1 => system_e}/system_power.json (96%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r1/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r1/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r1/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r1/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r1/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r1024/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r1024/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r1024/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r1024/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r1024/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r128/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r128/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r128/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r128/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r128/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r16/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r16/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r16/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r16/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r16/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r2/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r2/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r2/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r2/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r2/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r256/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r256/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r256/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r256/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r256/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r32/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r32/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r32/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r32/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r32/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r4/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r4/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r4/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r4/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r4/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r512/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r512/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r512/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r512/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r512/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r64/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r64/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r64/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r64/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r64/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r8/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r8/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r8/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r8/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/llama3-8b/r8/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_f/results/{sys_gaudi_dp2 => system_f}/system_power.json (97%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r1024/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r1024/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r1024/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r1024/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r1024/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r128/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r128/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r128/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r128/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r128/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r1536/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r1536/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r1536/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r1536/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r1536/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r192/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r192/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r192/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r192/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r192/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r2048/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r2048/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r2048/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r2048/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r2048/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r256/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r256/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r256/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r256/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r256/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r384/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r384/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r384/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r384/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r384/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r512/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r512/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r512/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r512/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r512/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r64/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r64/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r64/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r64/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r64/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r768/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r768/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r768/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r768/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/llama3-70b/r768/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_g/results/{sys_8gpu_vllm => system_g}/system_power.json (96%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r1024/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r1024/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r1024/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r1024/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r1024/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r128/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r128/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r128/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r128/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r128/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r1536/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r1536/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r1536/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r1536/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r1536/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r192/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r192/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r192/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r192/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r192/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r2048/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r2048/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r2048/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r2048/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r2048/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r256/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r256/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r256/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r256/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r256/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r384/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r384/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r384/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r384/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r384/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r512/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r512/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r512/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r512/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r512/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r64/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r64/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r64/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r64/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r64/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r768/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r768/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r768/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r768/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/llama3-70b/r768/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_h/results/{sys_8gpu_sglang => system_h}/system_power.json (96%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r1/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r1/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r1/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r1/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r1/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r128/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r128/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r128/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r128/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r128/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r16/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r16/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r16/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r16/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r16/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r2/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r2/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r2/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r2/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r2/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r256/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r256/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r256/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r256/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r256/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r32/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r32/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r32/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r32/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r32/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r4/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r4/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r4/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r4/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r4/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r512/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r512/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r512/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r512/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r512/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r64/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r64/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r64/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r64/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r64/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r8/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r8/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r8/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r8/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/deepseek-r1/r8/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_i/results/{sys_h200_8gpu => system_i}/system_power.json (96%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r1024/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r1024/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r1024/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r1024/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r1024/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r128/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r128/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r128/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r128/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r128/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r16384/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r16384/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r16384/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r16384/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r16384/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r2048/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r2048/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r2048/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r2048/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r2048/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r256/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r256/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r256/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r256/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r256/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r32/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r32/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r32/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r32/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r32/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r4096/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r4096/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r4096/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r4096/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r4096/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r512/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r512/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r512/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r512/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r512/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r64/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r64/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r64/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r64/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r64/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r8192/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r8192/config.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r8192/point.yaml (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r8192/result_summary.json (100%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/deepseek-r1/r8192/system_desc.json (98%) rename tests/fixtures/validation/submissions/sub_j/results/{sys_gb300_72gpu => system_j}/system_power.json (96%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r1000/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r1000/config.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r1000/point.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r1000/result_summary.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r1000/system_desc.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r16/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r16/config.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r16/point.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r16/result_summary.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r16/system_desc.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r20/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r20/config.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r20/point.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r20/result_summary.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r20/system_desc.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r256/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r256/config.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r256/point.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r256/result_summary.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r256/system_desc.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r512/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r512/config.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r512/point.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r512/result_summary.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r512/system_desc.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r768/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r768/config.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r768/point.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r768/result_summary.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r768/system_desc.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r88/accuracy_results.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r88/config.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r88/point.yaml (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r88/result_summary.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/llama3_1-8b/r88/system_desc.json (100%) rename tests/fixtures/validation/submissions/valid_standardized/results/{acme_h100x8_001 => system_k}/system_power.json (96%) diff --git a/tests/fixtures/validation/README.md b/tests/fixtures/validation/README.md new file mode 100644 index 000000000..3ef929d70 --- /dev/null +++ b/tests/fixtures/validation/README.md @@ -0,0 +1,25 @@ +# Validation fixtures + +The submission trees are synthetic test cases, not historical submissions or +measured results. Their accuracy scores, performance summaries, steady-state +windows, system configurations, and power disclosures are constructed to exercise +validation behavior. They must not be used to compare hardware or serving software. + +Each invented system has a neutral identifier (`system_a` through `system_l`). +Real model, dataset, accelerator, and processor names remain where they exercise +published policy keys and hardware power lookups; these names do not establish +that the configurations were run on those products. + +`valid_standardized` supplies a complete standardized submission layout. +`invalid_submission` exercises invalid declarations and missing requirements. +`sub_a` through `sub_j` exercise different curve, model, configuration, and power +paths. These are checker inputs: the directory names do not guarantee that every +case passes all rules in the current cohort. + +The corpus was adapted from the submission checker test corpus. Its fixture +regeneration tool explicitly describes the data as synthetic and constructs +accuracy values, steady-state windows, and power disclosures. The copies here +are maintained as native validation fixtures and require no external checker. + +`policy-fingerprints.json` pins the catalog and rule definitions independently of +these submission trees, so policy edits require an explicit expected-value update. diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/accuracy_results.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/accuracy_results.json rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/config.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/config.yaml rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/config.yaml diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/point.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/point.yaml rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/point.yaml diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/result_summary.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/result_summary.json rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/result_summary.json diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/system_desc.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/system_desc.json rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/system_desc.json index 2ba326397..4345d8064 100644 --- a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r16/system_desc.json +++ b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Invalid Corp", "submitter_contact": "contact@example.com", - "system_name": "bad_system", + "system_name": "system_l", "system_category": "datacenter", "max_supported_concurrency": 88, "serving_framework": "custom 0.0.1", diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/accuracy_results.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/accuracy_results.json rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/config.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/config.yaml rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/config.yaml diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/point.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/point.yaml rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/point.yaml diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/result_summary.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/result_summary.json rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/result_summary.json diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/system_desc.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/system_desc.json rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/system_desc.json index 7e874b293..6ee2cc387 100644 --- a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r38/system_desc.json +++ b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Invalid Corp", "submitter_contact": "contact@example.com", - "system_name": "bad_system", + "system_name": "system_l", "system_category": "datacenter", "max_supported_concurrency": 88, "serving_framework": "custom 0.0.1", diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/accuracy_results.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/accuracy_results.json rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/config.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/config.yaml rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/config.yaml diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/point.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/point.yaml rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/point.yaml diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/result_summary.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/result_summary.json rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/result_summary.json diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/system_desc.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/system_desc.json rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/system_desc.json index 980e8279b..5210d5257 100644 --- a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/Llama-3_1-8B-Instruct/r88/system_desc.json +++ b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Invalid Corp", "submitter_contact": "contact@example.com", - "system_name": "bad_system", + "system_name": "system_l", "system_category": "datacenter", "max_supported_concurrency": 88, "serving_framework": "custom 0.0.1", diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/system_power.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/system_power.json similarity index 96% rename from tests/fixtures/validation/submissions/invalid_submission/results/bad_system/system_power.json rename to tests/fixtures/validation/submissions/invalid_submission/results/system_l/system_power.json index ef7d5ad07..4fa3752e7 100644 --- a/tests/fixtures/validation/submissions/invalid_submission/results/bad_system/system_power.json +++ b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/system_power.json @@ -1,5 +1,5 @@ { - "system_desc_id": "bad_system", + "system_desc_id": "system_l", "cooling": "air", "node_sets": [ { diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/config.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/point.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/result_summary.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/system_desc.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/system_desc.json index f3d7279df..3e06b504a 100644 --- a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r1024/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-A", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_8gpu", + "system_name": "system_a", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/config.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/point.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/result_summary.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/system_desc.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/system_desc.json index 1abf0179c..81ae74864 100644 --- a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r128/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-A", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_8gpu", + "system_name": "system_a", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/config.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/point.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/result_summary.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/system_desc.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/system_desc.json index 8a0a841bc..92c85aa89 100644 --- a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r16/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-A", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_8gpu", + "system_name": "system_a", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/config.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/point.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/result_summary.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/system_desc.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/system_desc.json index 65992fcd6..5d6067749 100644 --- a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r2048/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-A", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_8gpu", + "system_name": "system_a", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/config.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/point.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/result_summary.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/system_desc.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/system_desc.json index 5e287b0e8..c496c2aac 100644 --- a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r4/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-A", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_8gpu", + "system_name": "system_a", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/config.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/point.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/result_summary.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/system_desc.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/system_desc.json index 994bd5df1..db77d10fc 100644 --- a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r512/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-A", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_8gpu", + "system_name": "system_a", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/config.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/point.yaml rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/result_summary.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/system_desc.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/system_desc.json index 0b97f12f6..41ffa1e8e 100644 --- a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/gpt-oss-120b/r64/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-A", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_8gpu", + "system_name": "system_a", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/system_power.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/system_power.json similarity index 96% rename from tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/system_power.json rename to tests/fixtures/validation/submissions/sub_a/results/system_a/system_power.json index 5d5bb073d..3109dbf5f 100644 --- a/tests/fixtures/validation/submissions/sub_a/results/sys_mi355x_8gpu/system_power.json +++ b/tests/fixtures/validation/submissions/sub_a/results/system_a/system_power.json @@ -1,5 +1,5 @@ { - "system_desc_id": "sys_mi355x_8gpu", + "system_desc_id": "system_a", "cooling": "air", "node_sets": [ { diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/config.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/point.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/result_summary.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/system_desc.json similarity index 97% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/system_desc.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/system_desc.json index a4963a2ea..f8099d87e 100644 --- a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r1024/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-B", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_16gpu", + "system_name": "system_b", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/config.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/point.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/result_summary.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/system_desc.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/system_desc.json index 8972b6334..4de583965 100644 --- a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r128/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-B", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_16gpu", + "system_name": "system_b", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/config.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/point.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/result_summary.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/system_desc.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/system_desc.json index 41d2cef5c..fefbabe16 100644 --- a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r16/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-B", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_16gpu", + "system_name": "system_b", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/config.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/point.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/result_summary.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/system_desc.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/system_desc.json index 3a0110165..fa81d1590 100644 --- a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r2048/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-B", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_16gpu", + "system_name": "system_b", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/config.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/point.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/result_summary.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/system_desc.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/system_desc.json index cc430f766..66eeeb0f6 100644 --- a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r4/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-B", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_16gpu", + "system_name": "system_b", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/config.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/point.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/result_summary.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/system_desc.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/system_desc.json index 620a0a0b8..c1b9445fc 100644 --- a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r512/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-B", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_16gpu", + "system_name": "system_b", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/config.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/point.yaml rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/result_summary.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/system_desc.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/system_desc.json index a34829f6d..adcc5e3b3 100644 --- a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/gpt-oss-120b/r64/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-B", "submitter_contact": "contact@example.com", - "system_name": "sys_mi355x_16gpu", + "system_name": "system_b", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/system_power.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/system_power.json similarity index 96% rename from tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/system_power.json rename to tests/fixtures/validation/submissions/sub_b/results/system_b/system_power.json index 79ae07df0..0db242472 100644 --- a/tests/fixtures/validation/submissions/sub_b/results/sys_mi355x_16gpu/system_power.json +++ b/tests/fixtures/validation/submissions/sub_b/results/system_b/system_power.json @@ -1,5 +1,5 @@ { - "system_desc_id": "sys_mi355x_16gpu", + "system_desc_id": "system_b", "cooling": "air", "node_sets": [ { diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/config.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/point.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/result_summary.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/system_desc.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/system_desc.json index 260d1aeeb..702c56e45 100644 --- a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r128/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-C", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_4chip", + "system_name": "system_c", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/config.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/point.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/result_summary.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/system_desc.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/system_desc.json index f646966d8..89c194f23 100644 --- a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r16/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-C", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_4chip", + "system_name": "system_c", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/config.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/point.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/result_summary.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/system_desc.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/system_desc.json index bd8840a07..afc25595f 100644 --- a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r256/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-C", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_4chip", + "system_name": "system_c", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/config.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/point.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/result_summary.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/system_desc.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/system_desc.json index dc4e3fb6d..d4d3e195c 100644 --- a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r32/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-C", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_4chip", + "system_name": "system_c", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/config.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/point.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/result_summary.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/system_desc.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/system_desc.json index ab52c2334..f0ae6e6b7 100644 --- a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r512/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-C", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_4chip", + "system_name": "system_c", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/config.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/point.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/result_summary.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/system_desc.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/system_desc.json index 42152ddcc..16837b117 100644 --- a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r64/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-C", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_4chip", + "system_name": "system_c", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/config.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/point.yaml rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/result_summary.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/system_desc.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/system_desc.json index 492bd9fd7..a25fa6d35 100644 --- a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/qwen3-coder-480b/r8/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-C", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_4chip", + "system_name": "system_c", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/system_power.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/system_power.json similarity index 96% rename from tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/system_power.json rename to tests/fixtures/validation/submissions/sub_c/results/system_c/system_power.json index 084553bb8..ba1355173 100644 --- a/tests/fixtures/validation/submissions/sub_c/results/sys_tpu_4chip/system_power.json +++ b/tests/fixtures/validation/submissions/sub_c/results/system_c/system_power.json @@ -1,5 +1,5 @@ { - "system_desc_id": "sys_tpu_4chip", + "system_desc_id": "system_c", "cooling": "liquid", "node_sets": [ { diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/config.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/point.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/result_summary.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/system_desc.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/system_desc.json index 9d3e336e4..2fc4ae5ce 100644 --- a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r1024/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-D", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_8chip", + "system_name": "system_d", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/config.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/point.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/result_summary.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/system_desc.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/system_desc.json index 872271340..4aa0a556b 100644 --- a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r128/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-D", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_8chip", + "system_name": "system_d", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/config.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/point.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/result_summary.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/system_desc.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/system_desc.json index e7d21f342..a14bb3639 100644 --- a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r16/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-D", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_8chip", + "system_name": "system_d", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/config.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/point.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/result_summary.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/system_desc.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/system_desc.json index 1ea55431b..f3bcab6aa 100644 --- a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r256/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-D", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_8chip", + "system_name": "system_d", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/config.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/point.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/result_summary.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/system_desc.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/system_desc.json index c5c549d00..d00bb00e3 100644 --- a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r32/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-D", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_8chip", + "system_name": "system_d", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/config.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/point.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/result_summary.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/system_desc.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/system_desc.json index bc660162a..17ad64718 100644 --- a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r512/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-D", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_8chip", + "system_name": "system_d", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/config.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/point.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/result_summary.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/system_desc.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/system_desc.json index 5f171ac62..3fa50bd10 100644 --- a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r64/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-D", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_8chip", + "system_name": "system_d", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/config.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/point.yaml rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/result_summary.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/system_desc.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/system_desc.json index c33ccc8a7..3b46b9774 100644 --- a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/qwen3-coder-480b/r8/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-D", "submitter_contact": "contact@example.com", - "system_name": "sys_tpu_8chip", + "system_name": "system_d", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM (TPU)", diff --git a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/system_power.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/system_power.json similarity index 96% rename from tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/system_power.json rename to tests/fixtures/validation/submissions/sub_d/results/system_d/system_power.json index 5f6438327..f7b7ebfb4 100644 --- a/tests/fixtures/validation/submissions/sub_d/results/sys_tpu_8chip/system_power.json +++ b/tests/fixtures/validation/submissions/sub_d/results/system_d/system_power.json @@ -1,5 +1,5 @@ { - "system_desc_id": "sys_tpu_8chip", + "system_desc_id": "system_d", "cooling": "liquid", "node_sets": [ { diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/config.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/point.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/result_summary.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/system_desc.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/system_desc.json index 2345b5af3..788d18396 100644 --- a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-E", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp1", + "system_name": "system_e", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/config.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/point.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/result_summary.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/system_desc.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/system_desc.json index 364f967cb..8cec9559c 100644 --- a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r1024/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-E", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp1", + "system_name": "system_e", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/config.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/point.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/result_summary.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/system_desc.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/system_desc.json index 146f52163..70388ed1c 100644 --- a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r128/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-E", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp1", + "system_name": "system_e", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/config.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/point.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/result_summary.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/system_desc.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/system_desc.json index d451053ae..338f336ee 100644 --- a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r16/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-E", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp1", + "system_name": "system_e", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/config.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/point.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/result_summary.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/system_desc.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/system_desc.json index 477083217..311fc4e89 100644 --- a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r2/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-E", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp1", + "system_name": "system_e", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/config.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/point.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/result_summary.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/system_desc.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/system_desc.json index 4f3a56426..166bcd6de 100644 --- a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r256/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-E", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp1", + "system_name": "system_e", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/config.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/point.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/result_summary.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/system_desc.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/system_desc.json index 6af50687a..829498fbd 100644 --- a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r32/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-E", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp1", + "system_name": "system_e", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/config.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/point.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/result_summary.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/system_desc.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/system_desc.json index cd43cda38..c8cff3cce 100644 --- a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r4/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-E", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp1", + "system_name": "system_e", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/config.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/point.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/result_summary.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/system_desc.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/system_desc.json index eccdac0a3..f7733fbdb 100644 --- a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r512/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-E", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp1", + "system_name": "system_e", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/config.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/point.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/result_summary.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/system_desc.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/system_desc.json index f236ea7d7..14c12df6f 100644 --- a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r64/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-E", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp1", + "system_name": "system_e", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/config.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/point.yaml rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/result_summary.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/system_desc.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/system_desc.json index 69d3fab9a..659a7445e 100644 --- a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/llama3-8b/r8/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-E", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp1", + "system_name": "system_e", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/system_power.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/system_power.json similarity index 96% rename from tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/system_power.json rename to tests/fixtures/validation/submissions/sub_e/results/system_e/system_power.json index 3d133a620..1cd446598 100644 --- a/tests/fixtures/validation/submissions/sub_e/results/sys_gaudi_dp1/system_power.json +++ b/tests/fixtures/validation/submissions/sub_e/results/system_e/system_power.json @@ -1,5 +1,5 @@ { - "system_desc_id": "sys_gaudi_dp1", + "system_desc_id": "system_e", "cooling": "air", "node_sets": [ { diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/config.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/point.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/result_summary.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/system_desc.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/system_desc.json index 25be44d22..d9689a269 100644 --- a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-F", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp2", + "system_name": "system_f", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/config.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/point.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/result_summary.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/system_desc.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/system_desc.json index 7590b1554..7cd1950a4 100644 --- a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r1024/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-F", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp2", + "system_name": "system_f", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/config.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/point.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/result_summary.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/system_desc.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/system_desc.json index f534f1db0..d2febe857 100644 --- a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r128/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-F", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp2", + "system_name": "system_f", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/config.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/point.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/result_summary.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/system_desc.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/system_desc.json index 698421adf..2399da1c5 100644 --- a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r16/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-F", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp2", + "system_name": "system_f", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/config.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/point.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/result_summary.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/system_desc.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/system_desc.json index af2bdc05e..9be0512be 100644 --- a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r2/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-F", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp2", + "system_name": "system_f", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/config.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/point.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/result_summary.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/system_desc.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/system_desc.json index a52613032..efac285be 100644 --- a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r256/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-F", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp2", + "system_name": "system_f", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/config.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/point.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/result_summary.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/system_desc.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/system_desc.json index 7299e888c..a8747d0c0 100644 --- a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r32/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-F", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp2", + "system_name": "system_f", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/config.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/point.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/result_summary.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/system_desc.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/system_desc.json index a53c325dd..7f555ed59 100644 --- a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r4/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-F", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp2", + "system_name": "system_f", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/config.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/point.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/result_summary.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/system_desc.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/system_desc.json index 14a208ef5..ddb8e4f05 100644 --- a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r512/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-F", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp2", + "system_name": "system_f", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/config.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/point.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/result_summary.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/system_desc.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/system_desc.json index 2971eb8aa..abb424f50 100644 --- a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r64/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-F", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp2", + "system_name": "system_f", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/config.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/point.yaml rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/result_summary.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/system_desc.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/system_desc.json index 3a351b933..a8e80717c 100644 --- a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/llama3-8b/r8/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-F", "submitter_contact": "contact@example.com", - "system_name": "sys_gaudi_dp2", + "system_name": "system_f", "system_category": "datacenter", "max_supported_concurrency": 1024, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/system_power.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/system_power.json similarity index 97% rename from tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/system_power.json rename to tests/fixtures/validation/submissions/sub_f/results/system_f/system_power.json index 7306e7adb..21550a655 100644 --- a/tests/fixtures/validation/submissions/sub_f/results/sys_gaudi_dp2/system_power.json +++ b/tests/fixtures/validation/submissions/sub_f/results/system_f/system_power.json @@ -1,5 +1,5 @@ { - "system_desc_id": "sys_gaudi_dp2", + "system_desc_id": "system_f", "cooling": "air", "node_sets": [ { diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/config.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/point.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/result_summary.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/system_desc.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/system_desc.json index b4218ddb6..e1b7473e2 100644 --- a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1024/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-G", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_vllm", + "system_name": "system_g", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/config.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/point.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/result_summary.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/system_desc.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/system_desc.json index 2bc9f37c3..d2cf70a38 100644 --- a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r128/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-G", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_vllm", + "system_name": "system_g", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/config.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/point.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/result_summary.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/system_desc.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/system_desc.json index 44567a339..720ed8eed 100644 --- a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r1536/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-G", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_vllm", + "system_name": "system_g", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/config.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/point.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/result_summary.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/system_desc.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/system_desc.json index 2d1302929..776439f6c 100644 --- a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r192/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-G", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_vllm", + "system_name": "system_g", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/config.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/point.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/result_summary.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/system_desc.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/system_desc.json index 90e2891d6..093b5cb06 100644 --- a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r2048/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-G", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_vllm", + "system_name": "system_g", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/config.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/point.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/result_summary.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/system_desc.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/system_desc.json index 3881de41f..a44a44683 100644 --- a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r256/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-G", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_vllm", + "system_name": "system_g", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/config.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/point.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/result_summary.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/system_desc.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/system_desc.json index 5f75c0c76..8365da8b4 100644 --- a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r384/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-G", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_vllm", + "system_name": "system_g", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/config.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/point.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/result_summary.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/system_desc.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/system_desc.json index 3429cd1e1..1ef2810de 100644 --- a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r512/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-G", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_vllm", + "system_name": "system_g", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/config.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/point.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/result_summary.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/system_desc.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/system_desc.json index dae732709..26b278e07 100644 --- a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r64/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-G", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_vllm", + "system_name": "system_g", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/config.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/point.yaml rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/result_summary.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/system_desc.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/system_desc.json index bb5740b64..2fc2d7a6e 100644 --- a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/llama3-70b/r768/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-G", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_vllm", + "system_name": "system_g", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "vLLM", diff --git a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/system_power.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/system_power.json similarity index 96% rename from tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/system_power.json rename to tests/fixtures/validation/submissions/sub_g/results/system_g/system_power.json index 4d161c256..1ab0e9142 100644 --- a/tests/fixtures/validation/submissions/sub_g/results/sys_8gpu_vllm/system_power.json +++ b/tests/fixtures/validation/submissions/sub_g/results/system_g/system_power.json @@ -1,5 +1,5 @@ { - "system_desc_id": "sys_8gpu_vllm", + "system_desc_id": "system_g", "cooling": "air", "node_sets": [ { diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/config.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/point.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/result_summary.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/system_desc.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/system_desc.json index a21b45eef..da1ccbf2b 100644 --- a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1024/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-H", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_sglang", + "system_name": "system_h", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "SGLang", diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/config.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/point.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/result_summary.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/system_desc.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/system_desc.json index 68ad390a2..8651c6c07 100644 --- a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r128/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-H", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_sglang", + "system_name": "system_h", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "SGLang", diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/config.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/point.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/result_summary.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/system_desc.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/system_desc.json index 881ef25e5..8bd4f491b 100644 --- a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r1536/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-H", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_sglang", + "system_name": "system_h", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "SGLang", diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/config.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/point.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/result_summary.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/system_desc.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/system_desc.json index fd90f338e..ff9566c99 100644 --- a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r192/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-H", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_sglang", + "system_name": "system_h", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "SGLang", diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/config.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/point.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/result_summary.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/system_desc.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/system_desc.json index 99d06ae1a..6d37f526e 100644 --- a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r2048/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-H", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_sglang", + "system_name": "system_h", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "SGLang", diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/config.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/point.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/result_summary.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/system_desc.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/system_desc.json index 7ab582744..af9605513 100644 --- a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r256/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-H", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_sglang", + "system_name": "system_h", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "SGLang", diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/config.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/point.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/result_summary.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/system_desc.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/system_desc.json index 510e3b803..065fc68ec 100644 --- a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r384/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-H", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_sglang", + "system_name": "system_h", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "SGLang", diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/config.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/point.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/result_summary.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/system_desc.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/system_desc.json index 770d456fc..adf7eac1a 100644 --- a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r512/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-H", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_sglang", + "system_name": "system_h", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "SGLang", diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/config.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/point.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/result_summary.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/system_desc.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/system_desc.json index b577f4c01..d282fbc88 100644 --- a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r64/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-H", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_sglang", + "system_name": "system_h", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "SGLang", diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/config.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/point.yaml rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/result_summary.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/system_desc.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/system_desc.json index 1b077fff0..6512e5184 100644 --- a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/llama3-70b/r768/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-H", "submitter_contact": "contact@example.com", - "system_name": "sys_8gpu_sglang", + "system_name": "system_h", "system_category": "datacenter", "max_supported_concurrency": 2048, "serving_framework": "SGLang", diff --git a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/system_power.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/system_power.json similarity index 96% rename from tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/system_power.json rename to tests/fixtures/validation/submissions/sub_h/results/system_h/system_power.json index c1490b516..f5baab29a 100644 --- a/tests/fixtures/validation/submissions/sub_h/results/sys_8gpu_sglang/system_power.json +++ b/tests/fixtures/validation/submissions/sub_h/results/system_h/system_power.json @@ -1,5 +1,5 @@ { - "system_desc_id": "sys_8gpu_sglang", + "system_desc_id": "system_h", "cooling": "air", "node_sets": [ { diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/config.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/point.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/result_summary.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/system_desc.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/system_desc.json index d534f9e1d..bd28e2c75 100644 --- a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r1/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-I", "submitter_contact": "contact@example.com", - "system_name": "sys_h200_8gpu", + "system_name": "system_i", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/config.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/point.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/result_summary.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/system_desc.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/system_desc.json index 973226972..00858a98f 100644 --- a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r128/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-I", "submitter_contact": "contact@example.com", - "system_name": "sys_h200_8gpu", + "system_name": "system_i", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/config.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/point.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/result_summary.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/system_desc.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/system_desc.json index f785137f6..1b5895a94 100644 --- a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r16/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-I", "submitter_contact": "contact@example.com", - "system_name": "sys_h200_8gpu", + "system_name": "system_i", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/config.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/point.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/result_summary.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/system_desc.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/system_desc.json index aea985de0..58c9bd606 100644 --- a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r2/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-I", "submitter_contact": "contact@example.com", - "system_name": "sys_h200_8gpu", + "system_name": "system_i", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/config.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/point.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/result_summary.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/system_desc.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/system_desc.json index 3c0bc3e26..04a8e2c14 100644 --- a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r256/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-I", "submitter_contact": "contact@example.com", - "system_name": "sys_h200_8gpu", + "system_name": "system_i", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/config.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/point.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/result_summary.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/system_desc.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/system_desc.json index 0e0a25fb4..fa5657e0d 100644 --- a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r32/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-I", "submitter_contact": "contact@example.com", - "system_name": "sys_h200_8gpu", + "system_name": "system_i", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/config.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/point.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/result_summary.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/system_desc.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/system_desc.json index a27ef4ed1..6a6d98ddf 100644 --- a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r4/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-I", "submitter_contact": "contact@example.com", - "system_name": "sys_h200_8gpu", + "system_name": "system_i", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/config.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/point.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/result_summary.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/system_desc.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/system_desc.json index d06b9647b..dc06f8e7e 100644 --- a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r512/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-I", "submitter_contact": "contact@example.com", - "system_name": "sys_h200_8gpu", + "system_name": "system_i", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/config.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/point.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/result_summary.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/system_desc.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/system_desc.json index bd61b0f41..4aa7cc486 100644 --- a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r64/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-I", "submitter_contact": "contact@example.com", - "system_name": "sys_h200_8gpu", + "system_name": "system_i", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/config.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/point.yaml rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/result_summary.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/system_desc.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/system_desc.json index bb73e7d98..61e1e3398 100644 --- a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/deepseek-r1/r8/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-I", "submitter_contact": "contact@example.com", - "system_name": "sys_h200_8gpu", + "system_name": "system_i", "system_category": "datacenter", "max_supported_concurrency": 512, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/system_power.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/system_power.json similarity index 96% rename from tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/system_power.json rename to tests/fixtures/validation/submissions/sub_i/results/system_i/system_power.json index dec21d8a6..4d902dc7d 100644 --- a/tests/fixtures/validation/submissions/sub_i/results/sys_h200_8gpu/system_power.json +++ b/tests/fixtures/validation/submissions/sub_i/results/system_i/system_power.json @@ -1,5 +1,5 @@ { - "system_desc_id": "sys_h200_8gpu", + "system_desc_id": "system_i", "cooling": "air", "node_sets": [ { diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/config.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/point.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/result_summary.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/system_desc.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/system_desc.json index 1e1930b42..a91ea8c6b 100644 --- a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r1024/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-J", "submitter_contact": "contact@example.com", - "system_name": "sys_gb300_72gpu", + "system_name": "system_j", "system_category": "datacenter", "max_supported_concurrency": 16384, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/config.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/point.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/result_summary.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/system_desc.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/system_desc.json index eaf0a9158..038d7bbbc 100644 --- a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r128/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-J", "submitter_contact": "contact@example.com", - "system_name": "sys_gb300_72gpu", + "system_name": "system_j", "system_category": "datacenter", "max_supported_concurrency": 16384, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/config.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/point.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/result_summary.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/system_desc.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/system_desc.json index 1ad0109a1..a47d5edd3 100644 --- a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r16384/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-J", "submitter_contact": "contact@example.com", - "system_name": "sys_gb300_72gpu", + "system_name": "system_j", "system_category": "datacenter", "max_supported_concurrency": 16384, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/config.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/point.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/result_summary.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/system_desc.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/system_desc.json index 3dc33743a..0cf7c26d4 100644 --- a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r2048/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-J", "submitter_contact": "contact@example.com", - "system_name": "sys_gb300_72gpu", + "system_name": "system_j", "system_category": "datacenter", "max_supported_concurrency": 16384, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/config.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/point.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/result_summary.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/system_desc.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/system_desc.json index 7f3422cff..136c33fac 100644 --- a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r256/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-J", "submitter_contact": "contact@example.com", - "system_name": "sys_gb300_72gpu", + "system_name": "system_j", "system_category": "datacenter", "max_supported_concurrency": 16384, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/config.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/point.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/result_summary.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/system_desc.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/system_desc.json index f3715be50..d14e6a1df 100644 --- a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r32/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-J", "submitter_contact": "contact@example.com", - "system_name": "sys_gb300_72gpu", + "system_name": "system_j", "system_category": "datacenter", "max_supported_concurrency": 16384, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/config.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/point.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/result_summary.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/system_desc.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/system_desc.json index 94ba28ba6..4b9e2c60f 100644 --- a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r4096/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-J", "submitter_contact": "contact@example.com", - "system_name": "sys_gb300_72gpu", + "system_name": "system_j", "system_category": "datacenter", "max_supported_concurrency": 16384, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/config.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/point.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/result_summary.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/system_desc.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/system_desc.json index 43a2259bc..48214dae4 100644 --- a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r512/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-J", "submitter_contact": "contact@example.com", - "system_name": "sys_gb300_72gpu", + "system_name": "system_j", "system_category": "datacenter", "max_supported_concurrency": 16384, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/config.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/point.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/result_summary.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/system_desc.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/system_desc.json index 37588a256..ce9b3c3ed 100644 --- a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r64/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-J", "submitter_contact": "contact@example.com", - "system_name": "sys_gb300_72gpu", + "system_name": "system_j", "system_category": "datacenter", "max_supported_concurrency": 16384, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/accuracy_results.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/config.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/config.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/point.yaml rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/point.yaml diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/result_summary.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/result_summary.json diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/system_desc.json similarity index 98% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/system_desc.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/system_desc.json index a23ab44fc..fdaec79c5 100644 --- a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/deepseek-r1/r8192/system_desc.json +++ b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/system_desc.json @@ -1,7 +1,7 @@ { "submitter_org_names": "Submitter-J", "submitter_contact": "contact@example.com", - "system_name": "sys_gb300_72gpu", + "system_name": "system_j", "system_category": "datacenter", "max_supported_concurrency": 16384, "serving_framework": "TensorRT-LLM", diff --git a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/system_power.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/system_power.json similarity index 96% rename from tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/system_power.json rename to tests/fixtures/validation/submissions/sub_j/results/system_j/system_power.json index 0287518fc..142b43500 100644 --- a/tests/fixtures/validation/submissions/sub_j/results/sys_gb300_72gpu/system_power.json +++ b/tests/fixtures/validation/submissions/sub_j/results/system_j/system_power.json @@ -1,5 +1,5 @@ { - "system_desc_id": "sys_gb300_72gpu", + "system_desc_id": "system_j", "cooling": "liquid", "node_sets": [ { diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/accuracy_results.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/config.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/config.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/point.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/point.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/result_summary.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/result_summary.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/system_desc.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r1000/system_desc.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/system_desc.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/accuracy_results.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/config.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/config.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/point.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/point.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/result_summary.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/result_summary.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/system_desc.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/system_desc.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/system_desc.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/accuracy_results.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/config.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/config.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/point.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/point.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/result_summary.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/result_summary.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/system_desc.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r20/system_desc.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/system_desc.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/accuracy_results.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/config.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/config.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/point.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/point.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/result_summary.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/result_summary.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/system_desc.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r256/system_desc.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/system_desc.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/accuracy_results.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/config.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/config.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/point.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/point.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/result_summary.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/result_summary.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/system_desc.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r512/system_desc.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/system_desc.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/accuracy_results.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/config.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/config.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/point.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/point.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/result_summary.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/result_summary.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/system_desc.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r768/system_desc.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/system_desc.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/accuracy_results.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/accuracy_results.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/accuracy_results.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/config.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/config.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/config.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/point.yaml similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/point.yaml rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/point.yaml diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/result_summary.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/result_summary.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/result_summary.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/system_desc.json similarity index 100% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r88/system_desc.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/system_desc.json diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/system_power.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/system_power.json similarity index 96% rename from tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/system_power.json rename to tests/fixtures/validation/submissions/valid_standardized/results/system_k/system_power.json index 2766647e6..1d5e9a43e 100644 --- a/tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/system_power.json +++ b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/system_power.json @@ -1,5 +1,5 @@ { - "system_desc_id": "acme_h100x8_001", + "system_desc_id": "system_k", "cooling": "air", "node_sets": [ { diff --git a/tests/unit/validation/test_policy_contracts.py b/tests/unit/validation/test_policy_contracts.py index 395d8ceba..0d72883f7 100644 --- a/tests/unit/validation/test_policy_contracts.py +++ b/tests/unit/validation/test_policy_contracts.py @@ -289,7 +289,7 @@ def test_power_cooling_must_match_description( policy = load_policy(bundled_policy_path()) source = ( Path(__file__).resolve().parents[3] - / "tests/fixtures/validation/submissions/valid_standardized/results/acme_h100x8_001/llama3_1-8b/r16/system_desc.json" + / "tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/system_desc.json" ) system = tmp_path / "results" / "system" point = system / "gpt-oss-120b" / "r16" From 131fe1b293862e2c1c8c9c1c2394bbdac463fe2c Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 05:27:53 -0700 Subject: [PATCH 14/18] refactor(validation): retain typed requirements and enforce approval identities --- docs/validation/DESIGN.md | 37 +- docs/validation/README.md | 137 +++--- .../validation/artifacts.py | 22 +- src/inference_endpoint/validation/bindings.py | 22 +- .../validation/checks/__init__.py | 4 +- .../validation/checks/accuracy.py | 120 ++--- .../validation/checks/collections.py | 104 ++--- .../validation/checks/datasets.py | 45 +- .../validation/checks/helpers.py | 46 -- .../validation/checks/metrics.py | 136 +++--- .../validation/checks/power.py | 58 +-- .../validation/checks/seeds.py | 33 +- .../{drafters.py => spec_decode_heads.py} | 68 +-- .../validation/checks/steady_state.py | 24 +- .../validation/checks/structure.py | 142 +++--- .../validation/checks/warmup.py | 28 +- .../validation/evidence/loaders.py | 55 ++- .../validation/evidence/point_config.py | 107 ++++- .../validation/evidence/point_summary.py | 14 +- .../validation/json_values.py | 24 + src/inference_endpoint/validation/models.py | 3 +- .../validation/operations.py | 114 ++++- src/inference_endpoint/validation/planner.py | 21 +- .../policies/2026-10-C1/catalog.yaml | 12 +- .../policies/2026-10-C1/point_checks.yaml | 28 +- .../2026-10-C1/submission_checks.yaml | 7 - .../policies/2026-10-C1/system_checks.yaml | 7 + .../validation/power/models.py | 4 +- .../validation/references.py | 4 +- .../validation/requirements.py | 16 + .../validation/schemas/requirements_v1.py | 301 ++++++++----- .../validation/schemas/v1.py | 6 +- .../validation/schemas/values_v1.py | 17 +- src/inference_endpoint/validation/types.py | 6 +- .../validation/vocabulary.py | 2 +- .../validation/policy-fingerprints.json | 150 +++---- tests/unit/validation/test_engine.py | 34 +- .../test_final_review_regressions.py | 354 +++++++++++++++ tests/unit/validation/test_loader.py | 17 +- tests/unit/validation/test_planner.py | 6 +- .../unit/validation/test_policy_contracts.py | 34 +- .../validation/test_review_regressions.py | 9 +- .../unit/validation/test_spec_decode_heads.py | 409 ++++++++++++++++++ .../validation/test_special_evaluators.py | 77 ++-- 44 files changed, 2053 insertions(+), 811 deletions(-) rename src/inference_endpoint/validation/checks/{drafters.py => spec_decode_heads.py} (50%) create mode 100644 src/inference_endpoint/validation/json_values.py create mode 100644 src/inference_endpoint/validation/requirements.py create mode 100644 tests/unit/validation/test_final_review_regressions.py create mode 100644 tests/unit/validation/test_spec_decode_heads.py diff --git a/docs/validation/DESIGN.md b/docs/validation/DESIGN.md index c41f491e9..ee42a565d 100644 --- a/docs/validation/DESIGN.md +++ b/docs/validation/DESIGN.md @@ -89,8 +89,12 @@ Each parser declares a cohort and inclusive revision interval. `schemas/requirements_v1.py` defines a typed contract for each check kind, including strict flags, numeric bounds, nested operands, and operation-specific required fields. Base rules and fully merged overrides must satisfy those -contracts before planning. `operations.py` shares operation enums between -contracts and evaluators. Overlapping intervals +contracts before planning. Validated requirement models remain immutable through +planning and execution; evaluators read typed attributes. Overrides are revalidated +against the same contract. Override patches contain only changed fields; each fully +merged rule is validated when loading and when planning. `operations.py` shares +operation and mode enums between +contracts and evaluators; unknown modes are rejected while loading the bundle. Overlapping intervals and unsupported releases fail explicitly. Revision 1 is supported. ## Classification and planning @@ -103,7 +107,34 @@ accept finite numbers and finite numeric strings; booleans, malformed values, and explicitly empty scores produce structural errors. Supplied non-null decode head declarations select approval checks even when identity is incomplete. Cooling uses the shared `Cooling` enum, including `mixed`, and requires a -matching policy overhead. +matching policy overhead. Artifact metrics must be finite. Calculated metrics are +checked for finiteness even when no stored metric is available. Cyclic aliases and +structures deeper than 100 levels produce artifact errors; shared aliases are valid. + +Comparisons accept typed field, constant, and sum operands. A field operand may +specify a numeric default for an absent value. Sample accounting compares the sum +of completed and failed samples with issued samples; omitted failed counts default +to zero. System names must agree within each submitted system. + +Agentic interactivity uses explicit output-token and elapsed-turn totals when +available. When both are absent and no samples failed, it uses the client's output +sequence total and latency total in nanoseconds. A partially supplied explicit pair +does not use that fallback. + +Speculative decoding heads are approved by model and identity. Declarations +may use an approved repository/revision pair or a full lowercase `git-sha1:` weight +checksum. Multiple supplied identity fields must agree. The catalog records each +head's approval cohort; target cohorts must satisfy the configured lead time. +Configuration-based identities require a target checksum (`git-sha1:` or `sha256:`) +and an exact, nonempty configuration dictionary. Equality preserves scalar types +at every nesting level: booleans, integers, and floats are distinct. Dictionary +key order is irrelevant; list order and length must match. Non-finite numbers +never match an approval identity. The same comparison enforces agreement between +`spec_decode_head` and its accepted input alias. Configuration identities cannot be combined with +weight identity fields. Catalog entries use `target_checksum` and `configuration` +in place of `repository` and `revision`, with the same model and approval-cohort +checks. This identity form is supported, but no configuration-based entries have +been officially approved; the bundled catalog contains only weight-based entries. `PointArtifacts.from_evidence` constructs a complete point with parsed evidence and a required classification. Calculated point power lives in `PointDerived`. diff --git a/docs/validation/README.md b/docs/validation/README.md index a4e3e8df3..381744741 100644 --- a/docs/validation/README.md +++ b/docs/validation/README.md @@ -33,12 +33,14 @@ names and references are validated by the versioned policy parser. - Dataset entries default to `is_legacy: false` and `sample_unit: sample` when those fields are omitted. - Seeds come from the bundled published cohort catalog. +- Speculative-head approvals include their publication cohorts. Weight-based and + configuration-based identities are supported; the published catalog currently + contains only weight-based approvals. ## Unresolved policy inputs The approved_client_revisions list is empty. Published full client SHAs must be supplied before client revision approval can pass. -Speculative-head approval publication cohorts are unspecified. The policy catalog is proposed, not an official MLCommons publication. ## Evaluator kinds @@ -66,7 +68,7 @@ The policy catalog is proposed, not an official MLCommons publication. - `field_constraints`: requires fields to be absent or exactly equal to prescribed values, including model-specific constraints. - `numeric_validity`: checks numeric type, presence, finiteness, and positivity. - - `comparison`: compares artifact values or constants using equality, + - `comparison`: compares artifact values, constants, or sums using equality, greater-than, or greater-than-or-equal. - `consistency`: checks equality between two values or across a collection of declarations, optionally excluding specified fields. @@ -110,8 +112,11 @@ The policy catalog is proposed, not an official MLCommons publication. - `artifact_binding`: - Matches the submitted client SHA against the approved list. - Matches checkpoint repository and revision against the model's approvals. - - `drafter_binding`: - - Matches a speculative decoding head by model, repository, and revision. + - `spec_decode_head`: + - Matches a speculative decoding head by model and approved weight identity + (repository/revision or `git-sha1:` checksum). + - Supports target-checksum plus exact configuration identities; none have + been officially approved, so the bundled catalog contains none. - Checks its approval lead time in cohorts. - `seed_binding`: - Checks published seed-set membership. @@ -147,14 +152,13 @@ Rules are grouped by scope; each group lists its policy file once. Policy file: `submission_checks.yaml`. -| Rule ID | Evaluator kind | -| ------------------------- | ------------------- | -| `accuracy-present` | `accuracy_presence` | -| `path-exists` | `presence` | -| `required-dir` | `presence` | -| `src-dir` | `presence` | -| `system-name-consistency` | `consistency` | -| `system-results-dir` | `presence` | +| Rule ID | Evaluator kind | +| -------------------- | ------------------- | +| `accuracy-present` | `accuracy_presence` | +| `path-exists` | `presence` | +| `required-dir` | `presence` | +| `src-dir` | `presence` | +| `system-results-dir` | `presence` | ### Implementation @@ -174,6 +178,7 @@ Policy file: `system_checks.yaml`. | `power-descriptor` | `power` | | `power-estimated` | `power` | | `system-description-across-curves` | `consistency` | +| `system-name-consistency` | `consistency` | ### Pareto curve @@ -214,62 +219,62 @@ Policy file: `curve_checks.yaml`. Policy file: `point_checks.yaml`. -| Rule ID | Evaluator kind | -| ---------------------------------- | ------------------- | -| `accuracy-valid` | `artifact_schema` | -| `agentic-metric-consistency` | `derived_metric` | -| `approved-checkpoint` | `artifact_binding` | -| `approved-drafter` | `drafter_binding` | -| `concurrency-in-range` | `region_placement` | -| `drafter-approval-lead-time` | `drafter_binding` | -| `endpoints-client-sha` | `artifact_binding` | -| `load-pattern` | `issuance` | -| `maximal-engagement` | `power` | -| `metric-consistency-accounting` | `comparison` | -| `metric-consistency-duration` | `comparison` | -| `metric-consistency-output-tokens` | `comparison` | -| `metric-consistency-system-tps` | `derived_metric` | -| `metric-consistency-tpot-p90` | `numeric_validity` | -| `metric-consistency-tps-per-kw` | `derived_metric` | -| `metric-consistency-tps-per-user` | `derived_metric` | -| `min-completed-samples` | `count` | -| `nodes-used` | `power` | -| `offline-declared` | `membership` | -| `point-config-valid` | `artifact_schema` | -| `point-dirname-concurrency` | `consistency` | -| `point-disclosure-complete` | `disclosure` | -| `point-duration` | `duration` | -| `point-power` | `power` | -| `point-rules-skipped` | `report` | -| `region-declared` | `membership` | -| `region-placement` | `region_placement` | -| `result-file-valid` | `artifact_schema` | -| `result-summary-present` | `presence` | -| `seed-config-legacy` | `seed_binding` | -| `seed-runtime-match` | `seed_binding` | -| `seed-set-adoption` | `seed_binding` | -| `seed-set-membership` | `seed_binding` | -| `shared-path-resolution` | `path_resolution` | -| `steady-state-basis` | `steady_state` | -| `steady-state-consistency` | `steady_state` | -| `steady-state-valid` | `steady_state` | -| `streaming-config` | `report` | -| `submission-flags` | `field_constraints` | -| `swebench-instance-count` | `count` | -| `swebench-template` | `field_constraints` | -| `system-description-present` | `presence` | -| `system-description-valid` | `artifact_schema` | -| `target-cohort` | `cohort_identifier` | -| `trajectories-to-issue` | `count` | -| `warmup-logs-retained` | `warmup` | -| `warmup-present` | `presence` | -| `warmup-salt` | `warmup` | +| Rule ID | Evaluator kind | +| ------------------------------------- | ------------------- | +| `accuracy-valid` | `artifact_schema` | +| `agentic-metric-consistency` | `derived_metric` | +| `approved-checkpoint` | `artifact_binding` | +| `approved-spec-decode-head` | `spec_decode_head` | +| `concurrency-in-range` | `region_placement` | +| `endpoints-client-sha` | `artifact_binding` | +| `load-pattern` | `issuance` | +| `maximal-engagement` | `power` | +| `metric-consistency-accounting` | `comparison` | +| `metric-consistency-duration` | `comparison` | +| `metric-consistency-output-tokens` | `comparison` | +| `metric-consistency-system-tps` | `derived_metric` | +| `metric-consistency-tpot-p90` | `numeric_validity` | +| `metric-consistency-tps-per-kw` | `derived_metric` | +| `metric-consistency-tps-per-user` | `derived_metric` | +| `min-completed-samples` | `count` | +| `nodes-used` | `power` | +| `offline-declared` | `membership` | +| `point-config-valid` | `artifact_schema` | +| `point-dirname-concurrency` | `consistency` | +| `point-disclosure-complete` | `disclosure` | +| `point-duration` | `duration` | +| `point-power` | `power` | +| `point-rules-skipped` | `report` | +| `region-declared` | `membership` | +| `region-placement` | `region_placement` | +| `result-file-valid` | `artifact_schema` | +| `result-summary-present` | `presence` | +| `seed-config-legacy` | `seed_binding` | +| `seed-runtime-match` | `seed_binding` | +| `seed-set-adoption` | `seed_binding` | +| `seed-set-membership` | `seed_binding` | +| `shared-path-resolution` | `path_resolution` | +| `spec-decode-head-approval-lead-time` | `spec_decode_head` | +| `steady-state-basis` | `steady_state` | +| `steady-state-consistency` | `steady_state` | +| `steady-state-valid` | `steady_state` | +| `streaming-config` | `report` | +| `submission-flags` | `field_constraints` | +| `swebench-instance-count` | `count` | +| `swebench-template` | `field_constraints` | +| `system-description-present` | `presence` | +| `system-description-valid` | `artifact_schema` | +| `target-cohort` | `cohort_identifier` | +| `trajectories-to-issue` | `count` | +| `warmup-logs-retained` | `warmup` | +| `warmup-present` | `presence` | +| `warmup-salt` | `warmup` | ### Validator catalogs Policy file: `catalog.yaml`. -| Rule ID | Evaluator kind | -| ----------------------- | ------------------- | -| `drafter-list-registry` | `catalog_integrity` | -| `seed-set-registry` | `catalog_integrity` | +| Rule ID | Evaluator kind | +| -------------------------- | ------------------- | +| `spec-decode-head-catalog` | `catalog_integrity` | +| `seed-set-registry` | `catalog_integrity` | diff --git a/src/inference_endpoint/validation/artifacts.py b/src/inference_endpoint/validation/artifacts.py index 4bb528b3b..20431bb6e 100644 --- a/src/inference_endpoint/validation/artifacts.py +++ b/src/inference_endpoint/validation/artifacts.py @@ -26,6 +26,7 @@ from .models import Policy from .planner import PlannedCheck, plan_checks from .power.models import PowerEvidence +from .schemas.requirements_v1 import RegionBasisRequirements from .types import Declaration, EvidenceKey, OfflineMode, Scope from .vocabulary import CheckKind @@ -116,7 +117,7 @@ def from_evidence( declared=declared, speculative_decoding=( config.value.speculative_decoding is not None - or config.value.drafter is not None + or config.value.spec_decode_head is not None ) if config.value is not None else None, @@ -193,7 +194,8 @@ def throughput(self) -> float | None: tokens = self.get_summary("output_sequence_lengths.total") if duration is None or tokens is None or duration <= 0: return None - return tokens / (duration / 1e9) + seconds = duration / 1e9 + return tokens / seconds if seconds > 0 else None @dataclass @@ -246,8 +248,8 @@ def resolve( ) -> Any: if not isinstance(address, str): return address - if address == "catalogs.approved_sped_decode_heads": - return self.catalogs.values.get("approved_sped_decode_heads") + if address == "catalogs.approved_spec_decode_heads": + return self.catalogs.values.get("approved_spec_decode_heads") if address.startswith("catalogs."): return nested(self.policy.catalogs, address[9:]) if address == "enrollment.models": @@ -384,8 +386,8 @@ def load_artifacts(root: Path, policy: Policy) -> Artifacts: artifacts.catalogs.values["seed_sets"] = load_seed_sets() except (ValueError, OSError) as exc: artifacts.catalogs.errors["seed_sets"] = str(exc) - artifacts.catalogs.values["approved_sped_decode_heads"] = policy.catalogs.get( - "approved_sped_decode_heads" + artifacts.catalogs.values["approved_spec_decode_heads"] = policy.catalogs.get( + "approved_spec_decode_heads" ) for system_dir in directories(root / "results"): system_ids = [] @@ -419,8 +421,8 @@ def load_artifacts(root: Path, policy: Policy) -> Artifacts: available = set() if "seed_sets" in artifacts.catalogs.values: available.add(EvidenceKey.SEED_CATALOG) - if "approved_sped_decode_heads" in artifacts.catalogs.values: - available.add(EvidenceKey.APPROVED_SPED_DECODE_HEADS) + if "approved_spec_decode_heads" in artifacts.catalogs.values: + available.add(EvidenceKey.APPROVED_SPEC_DECODE_HEADS) point = PointArtifacts.from_evidence( path, evidence, frozenset(available) ) @@ -493,7 +495,9 @@ def _compute_regions(artifacts: Artifacts, curve: Path, ids: list[str]) -> None: ] if not plans: return - effective_clamps = {check.rule.requirements["upper_clamp"] for check in plans} + effective_clamps = { + check.requirements(RegionBasisRequirements).upper_clamp for check in plans + } if len(effective_clamps) != 1: artifacts.derived.region_errors[str(curve)] = ( "Conflicting region-basis partitions cannot share one set of boundaries" diff --git a/src/inference_endpoint/validation/bindings.py b/src/inference_endpoint/validation/bindings.py index acbd846ad..eccd39010 100644 --- a/src/inference_endpoint/validation/bindings.py +++ b/src/inference_endpoint/validation/bindings.py @@ -10,21 +10,27 @@ from .outcomes import result from .planner import PlannedCheck from .results import CheckResult +from .schemas.requirements_v1 import ( + ArtifactBindingRequirements, + CatalogIntegrityRequirements, +) from .vocabulary import CheckKind class ArtifactBinding(Evaluator, kind=CheckKind.ARTIFACT_BINDING): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - matching = BindingMatch(check.rule.requirements["matching"]) + req = check.requirements(ArtifactBindingRequirements) + matching = req.matching findings = [] for point in artifacts.members(check): if matching is BindingMatch.CHECKPOINT: - addresses = check.rule.requirements["sources"] + addresses = req.sources + assert addresses is not None repository, revision = ( artifacts.resolve(address, check, point) for address in addresses ) model = point.context.model_id - catalog = artifacts.resolve(check.rule.requirements["catalog"], check) + catalog = artifacts.resolve(req.catalog, check) approved = catalog.get(model, ()) if catalog is not None else () if not approved: findings.append( @@ -62,9 +68,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul ) ) continue - revision = artifacts.resolve( - check.rule.requirements["source"], check, point - ) + revision = artifacts.resolve(req.source, check, point) if not isinstance(revision, str) or not re.fullmatch( r"[0-9a-f]{40}", revision ): @@ -77,7 +81,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul ) ) continue - approved = artifacts.resolve(check.rule.requirements["catalog"], check) + approved = artifacts.resolve(req.catalog, check) if not approved: findings.append( result( @@ -105,11 +109,11 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class CatalogIntegrity(Evaluator, kind=CheckKind.CATALOG_INTEGRITY): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - address = check.rule.requirements["catalog"] + address = check.requirements(CatalogIntegrityRequirements).catalog catalog_name = ( "seed_sets" if address == "context.cohorts.seed_sets" - else "approved_sped_decode_heads" + else "approved_spec_decode_heads" ) problem = artifacts.catalogs.errors.get(catalog_name) catalog = artifacts.resolve(address, check) diff --git a/src/inference_endpoint/validation/checks/__init__.py b/src/inference_endpoint/validation/checks/__init__.py index 4b660d653..fc6f3be38 100644 --- a/src/inference_endpoint/validation/checks/__init__.py +++ b/src/inference_endpoint/validation/checks/__init__.py @@ -2,6 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 """Register built-in callable evaluators by their policy kind.""" -from . import accuracy, drafters, power, seeds, steady_state, warmup +from . import accuracy, power, seeds, spec_decode_heads, steady_state, warmup -__all__ = ["accuracy", "drafters", "power", "seeds", "steady_state", "warmup"] +__all__ = ["accuracy", "spec_decode_heads", "power", "seeds", "steady_state", "warmup"] diff --git a/src/inference_endpoint/validation/checks/accuracy.py b/src/inference_endpoint/validation/checks/accuracy.py index 23b4698aa..a355de4e0 100644 --- a/src/inference_endpoint/validation/checks/accuracy.py +++ b/src/inference_endpoint/validation/checks/accuracy.py @@ -6,7 +6,7 @@ from collections import defaultdict from collections.abc import Mapping -from typing import Any, cast +from typing import Any from pydantic import JsonValue @@ -14,12 +14,23 @@ from ..artifacts import Artifacts, PointArtifacts from ..evaluator_base import Evaluator -from ..operations import AccuracyOperation +from ..operations import ( + AccuracyMode, + AccuracyOperation, + AccuracyPresenceSource, + FractionConversion, + PolicyAction, +) from ..planner import PlannedCheck from ..results import CheckResult, Severity +from ..schemas.requirements_v1 import ( + AccuracyCoverageRequirements, + AccuracyGateRequirements, + AccuracyPresenceRequirements, +) from ..types import AccuracyKind, OfflineMode from ..vocabulary import CheckKind -from .helpers import finding, number, unsupported, validate_modes, warning +from .helpers import finding, number, unsupported, warning def accuracy_entries( @@ -63,17 +74,24 @@ def accuracy_band(artifacts: Artifacts, point: PointArtifacts) -> str | None: class AccuracyPresenceEvaluator(Evaluator, kind=CheckKind.ACCURACY_PRESENCE): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - accepted = check.rule.requirements["accept"] + accepted = check.requirements(AccuracyPresenceRequirements).accept curves = set() for point in artifacts.members(check): root, _ = accuracy_entries(point) standalone = (point.path / "accuracy_results.json").exists() if root and ( - (standalone and "standalone_accuracy_results" in accepted) - or (not standalone and "embedded_nonempty_accuracy_scores" in accepted) + ( + standalone + and AccuracyPresenceSource.STANDALONE_ACCURACY_RESULTS in accepted + ) + or ( + not standalone + and AccuracyPresenceSource.EMBEDDED_NONEMPTY_ACCURACY_SCORES + in accepted + ) ): curves.add(point.curve) - minimum = check.rule.requirements["minimum_models"] + minimum = check.requirements(AccuracyPresenceRequirements).minimum_models return [ finding( check, @@ -85,11 +103,8 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class AccuracyCoverageEvaluator(Evaluator, kind=CheckKind.ACCURACY_COVERAGE): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - params = check.rule.requirements - invalid = validate_modes(check) - if invalid is not None: - return invalid - bands = artifacts.resolve(params["mandatory_bands"], check) + params = check.requirements(AccuracyCoverageRequirements) + bands = artifacts.resolve(params.mandatory_bands, check) counts: defaultdict[str | None, int] = defaultdict(int) offline = False for point in artifacts.members(check): @@ -97,7 +112,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul if not root: continue band = artifacts.region(point) - if band != "margin" or params.get("margin_counts"): + if band != "margin" or params.margin_counts: counts[band] += 1 if ( point.concurrency is not None @@ -113,8 +128,8 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul output = [ finding( check, - counts[band] >= params["minimum_results_per_band"], - f"Accuracy band {band}: {counts[band]} results; requires {params['minimum_results_per_band']}", + counts[band] >= params.minimum_results_per_band, + f"Accuracy band {band}: {counts[band]} results; requires {params.minimum_results_per_band}", ) for band in bands ] @@ -124,7 +139,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul if p.get_config("offline") != OfflineMode.DEDICATED } if ( - params.get("single_turn_requires_offline_accuracy") + params.single_turn_requires_offline_accuracy and LoadPatternType.AGENTIC_INFERENCE not in patterns ): output.append( @@ -141,15 +156,9 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class AccuracyGateEvaluator(Evaluator, kind=CheckKind.ACCURACY_GATE): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - params = check.rule.requirements - invalid = validate_modes(check) - if invalid is not None: - return invalid - try: - operation = AccuracyOperation(cast(str, params.get("operation"))) - except ValueError: - return unsupported(check) - model_catalog = artifacts.resolve(params["models"], check) + params = check.requirements(AccuracyGateRequirements) + operation = params.operation + model_catalog = artifacts.resolve(params.models, check) model = ( model_catalog.get(check.subject.model_id, {}) if isinstance(model_catalog, Mapping) @@ -173,12 +182,12 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul ) ] if operation is AccuracyOperation.PER_POINT_RANGE: - bounds = artifacts.resolve(params["bounds"], check) + bounds = artifacts.resolve(params.bounds, check) if not isinstance(bounds, (tuple, list)) or len(bounds) != 2: return unsupported(check, "Accuracy range catalog unavailable") output = [] for point in points: - value = artifacts.resolve(params["source"], check, point) + value = artifacts.resolve(params.source, check, point) if not number(value): output.append( warning( @@ -190,9 +199,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul continue low, high = bounds passed = ( - low <= value <= high - if params.get("inclusive") - else low < value < high + low <= value <= high if params.inclusive else low < value < high ) output.append( finding( @@ -215,10 +222,13 @@ def _agentic( check: PlannedCheck, artifacts: Artifacts, points: list[PointArtifacts], - params: Mapping[str, Any], + params: AccuracyGateRequirements, operation: AccuracyOperation, ) -> list[CheckResult]: - minimum = artifacts.resolve(params["minimum"], check) + assert params.dataset is not None + assert params.input_range is not None + assert params.output_scale is not None + minimum = artifacts.resolve(params.minimum, check) if not number(minimum): return unsupported(check, "Accuracy minimum unavailable") output = [] @@ -228,20 +238,20 @@ def _agentic( selected = [ (name, metrics) for name, metrics in scores.items() - if dataset_matches(name, params["dataset"]) + if dataset_matches(name, params.dataset) ] if not selected: if operation is AccuracyOperation.PER_POINT_FRACTION: output.append( warning( - check, f"Missing {params['dataset']} accuracy result", point + check, f"Missing {params.dataset} accuracy result", point ) ) continue values = [] for name, metrics in selected: value = metrics.get("score", next(iter(metrics.values()), None)) - lo, hi = params["input_range"] + lo, hi = params.input_range if not number(value) or not lo <= value <= hi: output.append( finding( @@ -252,7 +262,7 @@ def _agentic( ) ) continue - values.append(value * params["output_scale"]) + values.append(value * params.output_scale) if not values: continue average = sum(values) / len(values) @@ -268,13 +278,14 @@ def _agentic( else: by_band[accuracy_band(artifacts, point)].append(average) if operation is AccuracyOperation.MEAN_OF_BAND_MEANS: - bands = artifacts.resolve(params["bands"], check) + assert params.required_band_count is not None + bands = artifacts.resolve(params.bands, check) available = [band for band in bands if by_band[band]] - if len(available) < params["required_band_count"]: + if len(available) < params.required_band_count: output.append( warning( check, - f"SWE-bench accuracy has {len(available)} populated bands; requires {params['required_band_count']}", + f"SWE-bench accuracy has {len(available)} populated bands; requires {params.required_band_count}", ) ) for band in available: @@ -304,18 +315,20 @@ def _single_turn( check: PlannedCheck, artifacts: Artifacts, points: list[PointArtifacts], - params: Mapping[str, Any], + params: AccuracyGateRequirements, profile: Mapping[str, Any], operation: AccuracyOperation, ) -> list[CheckResult]: - required = artifacts.resolve(params["required_datasets"], check) - datasets = artifacts.resolve(params["dataset_catalog"], check) + if operation is AccuracyOperation.ISSUED_COUNT: + assert params.minimum_repeats is not None + required = artifacts.resolve(params.required_datasets, check) + datasets = artifacts.resolve(params.dataset_catalog, check) metrics = profile.get("metrics", {}) if not required or not isinstance(datasets, Mapping): return unsupported(check, "Required accuracy datasets unavailable") if ( operation is AccuracyOperation.SINGLE_TURN_METRICS - and params.get("aggregation") != "sample_weighted_mean" + and params.aggregation != AccuracyMode.SAMPLE_WEIGHTED_MEAN ): return unsupported(check, "Unsupported accuracy aggregation") output = [] @@ -344,8 +357,9 @@ def _single_turn( ) continue if operation is AccuracyOperation.ISSUED_COUNT: + assert params.minimum_repeats is not None for ds in required: - threshold = datasets.get(ds, {}).get(params["threshold"]) + threshold = datasets.get(ds, {}).get(params.threshold) if not number(threshold): output.append( finding( @@ -362,11 +376,9 @@ def _single_turn( if not dataset_matches(name, ds): continue count = entry.get("num_samples") - repeats = entry.get("n_repeats", params["missing_repeats"]) + repeats = entry.get("n_repeats", params.missing_repeats) if number(count) and number(repeats): - counts.append( - count * max(repeats, params["minimum_repeats"]) - ) + counts.append(count * max(repeats, params.minimum_repeats)) if counts: total = sum(counts) output.append( @@ -384,7 +396,7 @@ def _single_turn( for metric, value in scores.get(name, {}).items(): pairs[ metric.lower() - if params.get("metric_matching") == "case_insensitive" + if params.metric_matching == AccuracyMode.CASE_INSENSITIVE else metric ].append((value, weight)) aggregate = { @@ -393,7 +405,7 @@ def _single_turn( if ( set(aggregate) == {"score"} and len(metrics) == 1 - and params.get("unnamed_scalar") == "sole_metric_only_with_warning" + and params.unnamed_scalar == AccuracyMode.SOLE_METRIC_ONLY_WITH_WARNING ): name = next(iter(metrics)) aggregate[name] = aggregate.pop("score") @@ -407,11 +419,11 @@ def _single_turn( for name, bounds in metrics.items(): score = aggregate.get( name.lower() - if params.get("metric_matching") == "case_insensitive" + if params.metric_matching == AccuracyMode.CASE_INSENSITIVE else name ) if score is None: - if params.get("on_missing_metric") != "skip": + if params.on_missing_metric != PolicyAction.SKIP: output.append( finding( check, @@ -447,8 +459,8 @@ def _single_turn( ) continue if ( - params.get("fraction_conversion") - == "legacy_score_and_threshold_heuristic" + params.fraction_conversion + == FractionConversion.LEGACY_VALUE_AND_THRESHOLD_HEURISTIC and 0 <= score <= 1 and lower is not None and lower > 1 diff --git a/src/inference_endpoint/validation/checks/collections.py b/src/inference_endpoint/validation/checks/collections.py index 851e146bf..94d86dcbf 100644 --- a/src/inference_endpoint/validation/checks/collections.py +++ b/src/inference_endpoint/validation/checks/collections.py @@ -5,43 +5,49 @@ from __future__ import annotations import math -from collections.abc import Mapping from pathlib import Path from inference_endpoint.config.schema import LoadPatternType from ..artifacts import Artifacts, PointArtifacts from ..evaluator_base import Evaluator -from ..operations import OfflineOperation, RegionPlacementOperation +from ..operations import OfflineOperation, PolicyAction, RegionPlacementOperation from ..outcomes import result from ..planner import PlannedCheck from ..results import CheckResult, Severity +from ..schemas.requirements_v1 import ( + Bounds, + CollectionSizeRequirements, + CoverageRequirements, + OfflineRequirements, + RegionBasisRequirements, + RegionBoundariesRequirements, + RegionPlacementRequirements, +) from ..types import AccuracyKind, OfflineMode from ..vocabulary import CheckKind class CollectionSize(Evaluator, kind=CheckKind.COLLECTION_SIZE): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements - count = len(artifacts.resolve(req["source"], check) or []) - minimum = req.get("minimum", 0) - maximum = req.get("maximum", math.inf) + req = check.requirements(CollectionSizeRequirements) + count = len(artifacts.resolve(req.source, check) or []) + minimum = req.minimum if req.minimum is not None else 0 + maximum = req.maximum if req.maximum is not None else math.inf members = artifacts.members(check) - for override in req.get("overrides", []): - when = override["when"] + for override in req.overrides if req.overrides is not None else []: + when = override.when agentic = any( p.get_config("runtime_settings.load_pattern") == LoadPatternType.AGENTIC_INFERENCE for p in members ) if ( - when.get("curve_type") == AccuracyKind.SINGLE_TURN + when.curve_type == AccuracyKind.SINGLE_TURN and not agentic - and any( - p.get_config("offline") == when.get("has_offline") for p in members - ) + and any(p.get_config("offline") == when.has_offline for p in members) ): - minimum = override.get("minimum", minimum) + minimum = override.minimum return [ result( check, @@ -64,17 +70,17 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul ] points = artifacts.members(check) concurrencies = [p.concurrency for p in points if p.concurrency is not None] - req = check.rule.requirements + req = check.requirements(RegionBasisRequirements) if not concurrencies: return [result(check, False, "No readable concurrency values")] finding = result( check, True, - f"C_min={min(min(concurrencies), req['upper_clamp'])} from {len(concurrencies)} points", + f"C_min={min(min(concurrencies), req.upper_clamp)} from {len(concurrencies)} points", ) if ( len(concurrencies) < len(points) - and req.get("on_partial_parse") == "warning" + and req.on_partial_parse == PolicyAction.WARNING ): finding = finding.model_copy(update={"severity": Severity.WARNING}) return [finding] @@ -82,17 +88,17 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class RegionBoundaries(Evaluator, kind=CheckKind.REGION_BOUNDARIES): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements + req = check.requirements(RegionBoundariesRequirements) cmax = artifacts.resolve("curve.max_supported_concurrency", check) points = artifacts.members(check) cs = [p.concurrency for p in points if p.concurrency is not None] - cmin = min(min(cs), req["require_c_min"][1]) if cs else None + cmin = min(min(cs), req.require_c_min[1]) if cs else None valid = ( cmin is not None - and req["require_c_min"][0] <= cmin <= req["require_c_min"][1] + and req.require_c_min[0] <= cmin <= req.require_c_min[1] and isinstance(cmax, (int, float)) and not isinstance(cmax, bool) - and cmax > req["require_c_max_greater_than"] + and cmax > req.require_c_max_greater_than and check.subject.id in artifacts.derived.regions ) return [ @@ -106,66 +112,55 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class Coverage(Evaluator, kind=CheckKind.COVERAGE): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements - band = req["band"] + req = check.requirements(CoverageRequirements) + band = req.band members = artifacts.members(check) def in_band(point: PointArtifacts) -> bool: if point.get_config("offline") == OfflineMode.DEDICATED: return False - if isinstance(band, Mapping): + if isinstance(band, Bounds): return ( point.concurrency is not None - and band["minimum"] <= point.concurrency <= band["maximum"] + and band.minimum <= point.concurrency <= band.maximum ) actual = artifacts.region(point) return ( actual == band or band == "high_concurrency" and actual == "margin" - and req.get("margin_counts", False) + and req.margin_counts ) count = sum(in_band(p) for p in members) return [ result( check, - count >= req["minimum_count"], - f"{band}: {count} points; requires {req['minimum_count']}", + count >= req.minimum_count, + f"{band}: {count} points; requires {req.minimum_count}", ) ] class RegionPlacement(Evaluator, kind=CheckKind.REGION_PLACEMENT): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements + req = check.requirements(RegionPlacementRequirements) point = artifacts.index.points[check.subject.id] actual = artifacts.region(point) - try: - operation = RegionPlacementOperation(req["operation"]) - except ValueError: - return [ - result( - check, - False, - f"Unsupported region-placement operation: {req['operation']}", - blocked=True, - ) - ] + operation = req.operation if operation is RegionPlacementOperation.IN_RANGE: return [ result( check, - actual is not None - and (req.get("include_margin") or actual != "margin"), + actual is not None and (req.include_margin or actual != "margin"), f"Concurrency {point.concurrency} classified as {actual}", ) ] declared = point.get_config("region") if ( - declared in req.get("ignore_declared", []) + declared in req.ignore_declared or (actual is None or declared is None) - and req.get("on_missing_or_unclassifiable") == "skip" + and req.on_missing_or_unclassifiable == PolicyAction.SKIP ): return [] return [ @@ -177,22 +172,12 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class Offline(Evaluator, kind=CheckKind.OFFLINE): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements + req = check.requirements(OfflineRequirements) points = artifacts.members(check) cmax = artifacts.resolve("curve.max_supported_concurrency", check) - try: - operation = OfflineOperation(req["operation"]) - except ValueError: - return [ - result( - check, - False, - f"Unsupported Offline operation: {req['operation']}", - blocked=True, - ) - ] + operation = req.operation if operation is OfflineOperation.DECLARATION_COUNT: - elected_concurrency = artifacts.resolve(req["elected_must_equal"], check) + elected_concurrency = artifacts.resolve(req.elected_must_equal, check) agentic = any( p.get_config("runtime_settings.load_pattern") == LoadPatternType.AGENTIC_INFERENCE @@ -204,7 +189,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul if p.get_config("offline") in (OfflineMode.ELECTED, OfflineMode.DEDICATED) ] - expected = req["agentic_count" if agentic else "single_turn_count"] + expected = req.agentic_count if agentic else req.single_turn_count findings = [ result( check, @@ -223,7 +208,8 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul if p.get_config("offline") == OfflineMode.ELECTED ) return findings - floor = artifacts.resolve(req["concurrency_floor"], check) + assert req.throughput_minimum_multiplier is not None + floor = artifacts.resolve(req.concurrency_floor, check) reference = next( ( p @@ -253,7 +239,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul and reference.throughput is not None and point.throughput is not None ): - minimum = reference.throughput * req["throughput_minimum_multiplier"] + minimum = reference.throughput * req.throughput_minimum_multiplier findings.append( result( check, diff --git a/src/inference_endpoint/validation/checks/datasets.py b/src/inference_endpoint/validation/checks/datasets.py index c98c0f8f6..a4ea7ee00 100644 --- a/src/inference_endpoint/validation/checks/datasets.py +++ b/src/inference_endpoint/validation/checks/datasets.py @@ -6,10 +6,14 @@ from ..artifacts import Artifacts from ..evaluator_base import Evaluator -from ..operations import CountOperator +from ..operations import CountOperator, PolicyAction from ..outcomes import result from ..planner import PlannedCheck from ..results import CheckResult +from ..schemas.requirements_v1 import ( + CountRequirements, + FieldConstraintsRequirements, +) from ..types import SampleUnit from ..vocabulary import CheckKind @@ -18,32 +22,34 @@ class Count(Evaluator, kind=CheckKind.COUNT): """Compare a reported or configured count with a catalog reference count.""" def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements + req = check.requirements(CountRequirements) dataset = ( - artifacts.resolve(req["dataset_source"], check) - if "dataset_source" in req - else req["dataset"] + artifacts.resolve(req.dataset_source, check) + if req.dataset_source is not None + else req.dataset ) - entry = (artifacts.resolve(req["catalog"], check) or {}).get(dataset, {}) - supported_units = req.get("supported_sample_units") + entry = (artifacts.resolve(req.catalog, check) or {}).get(dataset, {}) + supported_units = req.supported_sample_units if ( supported_units is not None and entry.get("sample_unit", SampleUnit.SAMPLE) not in supported_units ): - if req.get("on_unsupported_sample_unit") == "skip": + if req.on_unsupported_sample_unit == PolicyAction.SKIP: return [] return [result(check, False, "Unsupported sample unit", blocked=True)] - expected = entry.get(req.get("threshold", "sample_count")) + expected = entry.get( + req.threshold if req.threshold is not None else "sample_count" + ) if expected is None: - if req.get("on_unknown_or_null_threshold") == "skip": + if req.on_unknown_or_null_threshold == PolicyAction.SKIP: return [] return [result(check, False, "Reference count unavailable", blocked=True)] if expected <= 0: return [ result(check, False, "Reference count must be positive", blocked=True) ] - operation = CountOperator(req["operator"]) - sources = req.get("sources") or [req["source"]] + operation = req.operator + sources = req.sources or [req.source] findings = [] for source in sources: actual = artifacts.resolve(source, check) @@ -75,21 +81,22 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class FieldConstraints(Evaluator, kind=CheckKind.FIELD_CONSTRAINTS): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements + req = check.requirements(FieldConstraintsRequirements) point = artifacts.index.points[check.subject.id] - if "constraints_by_model" in req: - constraint = ( - artifacts.resolve(req["constraints_by_model"], check) or {} - ).get(check.subject.model_id) + if req.constraints_by_model is not None: + assert req.target is not None + constraint = (artifacts.resolve(req.constraints_by_model, check) or {}).get( + check.subject.model_id + ) if constraint is None: return [ result( check, False, "Model field constraint unavailable", blocked=True ) ] - constraints = {req["target"]: constraint} + constraints = {req.target: constraint} else: - constraints = artifacts.resolve(req["constraints"], check) + constraints = artifacts.resolve(req.constraints, check) findings = [] for field, constraint in constraints.items(): value = ( diff --git a/src/inference_endpoint/validation/checks/helpers.py b/src/inference_endpoint/validation/checks/helpers.py index ca99d5a2d..01c0f23a3 100644 --- a/src/inference_endpoint/validation/checks/helpers.py +++ b/src/inference_endpoint/validation/checks/helpers.py @@ -63,49 +63,3 @@ def block_value(point: PointArtifacts, name: str) -> dict[str, JsonValue] | None if value is not None else point.get_config(name) ) - - -def validate_modes(check: PlannedCheck) -> list[CheckResult] | None: - """Reject policy modes whose semantics this parser revision cannot execute.""" - allowed = { - "matching": {"model_and_repository_and_revision"}, - "on_unavailable": {"blocked"}, - "on_legacy_registry_without_cohorts": {"blocked"}, - "on_empty_catalog": {"error"}, - "on_missing_repository_or_revision": {"error"}, - "on_unmatched_drafter": {"skip"}, - "on_unparseable_cohort": {"warning"}, - "on_missing_approval_cohort": {"warning"}, - "on_undeclared_nodes": {"fully_engaged"}, - "declared_total_scaling": {"maximum_engaged_fraction"}, - "switch_scaling": {"engaged_node_fraction"}, - "on_spare_nodes": {"warning"}, - "on_missing_parallelism": {"warning"}, - "on_disaggregated": {"warning"}, - "on_unknown_accelerator_count": {"warning"}, - "on_missing_window_extent": {"skip_count_comparison"}, - "on_missing_or_nonofficial": {"warning"}, - "fallback": {"whole_run_total"}, - "on_drift": {"warn_range_or_slope_required"}, - "disabled_warmup": {"exempt"}, - "on_missing_weights": {"arithmetic_mean"}, - "on_missing_required_dataset": {"error"}, - "on_unrelated_dataset": {"exclude"}, - "on_no_sample_counts": {"skip"}, - "on_unknown_model": {"warning"}, - "on_unknown_or_unpublished": {"warning"}, - "on_missing_metric": {"skip", "error"}, - "fraction_conversion": {"legacy_value_and_threshold_heuristic"}, - "count_scope": {"per_required_dataset_or_suite"}, - "units": {"samples_times_repeats"}, - "score": {"score_or_first_metric"}, - "unnamed_scalar": {"sole_metric_only_with_warning"}, - "on_multiple_results_per_band": {"average_and_warn"}, - "on_missing_bands": {"gate_available_mean_and_warn"}, - "on_no_results": {"warning"}, - } - for field, modes in allowed.items(): - value = check.rule.requirements.get(field) - if value is not None and value not in modes: - return unsupported(check, f"Unsupported policy mode {field}={value!r}") - return None diff --git a/src/inference_endpoint/validation/checks/metrics.py b/src/inference_endpoint/validation/checks/metrics.py index 39e88471b..6641b9050 100644 --- a/src/inference_endpoint/validation/checks/metrics.py +++ b/src/inference_endpoint/validation/checks/metrics.py @@ -6,8 +6,6 @@ import math import operator -from collections.abc import Mapping -from typing import Any from ..artifacts import Artifacts from ..evaluator_base import Evaluator @@ -16,11 +14,21 @@ ComparisonOperator, DerivedOperation, DurationBasis, - OperandKind, + PolicyAction, ) from ..outcomes import result from ..planner import PlannedCheck from ..results import CheckResult +from ..schemas.requirements_v1 import ( + ComparisonRequirements, + ConstantOperand, + DerivedMetricRequirements, + DurationRequirements, + FieldOperand, + NumericValidityRequirements, + Operand, + SumOperand, +) from ..vocabulary import CheckKind @@ -32,71 +40,76 @@ class Comparison(Evaluator, kind=CheckKind.COMPARISON): } def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements - when = req.get("when") + req = check.requirements(ComparisonRequirements) + when = req.when if when: - actual = artifacts.resolve(when["field"], check) - if actual is None or actual <= when["greater_than"]: + actual = artifacts.resolve(when.field, check) + if actual is None or actual <= when.greater_than: return [] - def operand(value: Mapping[str, Any]) -> Any: - return ( - value["value"] - if OperandKind(value["kind"]) is OperandKind.CONSTANT - else artifacts.resolve(value["field"], check) - ) - - left = operand(req["left"]) - right = operand(req["right"]) - op = self.operations.get(ComparisonOperator(req["operator"])) - if op is None: - return [ - result( - check, - False, - f"Unsupported comparison {req['operator']}", - blocked=True, + def operand(value: Operand) -> int | float | None: + if isinstance(value, ConstantOperand): + return value.value + if isinstance(value, FieldOperand): + resolved = artifacts.resolve(value.field, check) + if resolved is None: + resolved = value.default + return ( + resolved + if isinstance(resolved, (int, float)) + and not isinstance(resolved, bool) + else None ) - ] + assert isinstance(value, SumOperand) + operands = [operand(child) for child in value.operands] + if any(child is None for child in operands): + return None + return sum(child for child in operands if child is not None) + + left = operand(req.left) + right = operand(req.right) + op = self.operations[req.operator] valid = ( left is not None and right is not None and not isinstance(left, bool) and not isinstance(right, bool) + and (not isinstance(left, float) or math.isfinite(left)) + and (not isinstance(right, float) or math.isfinite(right)) and op(left, right) ) - return [result(check, valid, f"{left!r} {req['operator']} {right!r}")] + return [result(check, valid, f"{left!r} {req.operator} {right!r}")] class NumericValidity(Evaluator, kind=CheckKind.NUMERIC_VALIDITY): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements - value = artifacts.resolve(req["source"], check) - if value is None and not req.get("require_present"): + req = check.requirements(NumericValidityRequirements) + value = artifacts.resolve(req.source, check) + if value is None and not req.require_present: return [] valid = isinstance(value, (int, float)) and not isinstance(value, bool) - if valid and req.get("finite"): + if valid and req.finite: valid = math.isfinite(value) - if valid and req.get("strictly_positive"): + if valid and req.strictly_positive: valid = value > 0 - return [result(check, valid, f"{req['source']}: {value!r}")] + return [result(check, valid, f"{req.source}: {value!r}")] class Duration(Evaluator, kind=CheckKind.DURATION): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: point = artifacts.index.points[check.subject.id] - req = check.rule.requirements + req = check.requirements(DurationRequirements) band = artifacts.region(point) - if band is None and req.get("on_unclassifiable_concurrency") == "skip": + if band is None and req.on_unclassifiable_concurrency == PolicyAction.SKIP: return [] - thresholds = artifacts.resolve(req["thresholds"], check) + thresholds = artifacts.resolve(req.thresholds, check) minimum = thresholds.get(band) duration = None - for basis in req["basis_precedence"]: + for basis in req.basis_precedence: if basis == DurationBasis.REPORTED_WINDOW_ISSUE_SPAN: value = point.get_config("steady_state.window.duration_s") if value is not None and ( - not req.get("window_status_required") + not req.window_status_required or point.get_config("steady_state.status") == SteadyStateStatus.WINDOWABLE ): @@ -124,17 +137,18 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class DerivedMetric(Evaluator, kind=CheckKind.DERIVED_METRIC): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements - operation = DerivedOperation(req["operation"]) + req = check.requirements(DerivedMetricRequirements) + operation = req.operation findings = [] points = artifacts.members(check) - constants = artifacts.resolve(req.get("constants"), check) or {} + constants = artifacts.resolve(req.constants, check) or {} if operation is DerivedOperation.CURVE_PEAK_RATIO: + assert req.tolerance is not None available = [ p for p in points if p.throughput is not None - and isinstance(artifacts.resolve(req["stored"], check, p), (int, float)) + and isinstance(artifacts.resolve(req.stored, check, p), (int, float)) ] peak = max( (p.throughput for p in available if p.throughput is not None), default=0 @@ -142,34 +156,46 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul if peak <= 0: return [] for point in available: - stored = artifacts.resolve(req["stored"], check, point) + stored = artifacts.resolve(req.stored, check, point) assert point.throughput is not None ratio = point.throughput / peak findings.append( result( check, math.isfinite(stored) - and abs(stored - ratio) <= req["tolerance"]["absolute"], + and abs(stored - ratio) <= req.tolerance.absolute, f"Utilization {stored}; derived {ratio}", point=point, ) ) return findings for point in points: - stored = artifacts.resolve(req.get("stored"), check, point) + stored = artifacts.resolve(req.stored, check, point) expected: float | None = None if operation is DerivedOperation.OUTPUT_TOKENS_PER_ELAPSED_SECOND: expected = point.throughput elif operation is DerivedOperation.INVERSE_TPOT_P90_MS: + assert req.numerator is not None tpot = artifacts.resolve( "result_summary.tpot.percentiles.90", check, point ) if isinstance(tpot, (int, float)) and math.isfinite(tpot) and tpot > 0: - expected = req["numerator"] / (tpot / 1e6) + milliseconds = tpot / 1e6 + if milliseconds <= 0: + findings.append( + result( + check, + False, + "TPOT duration is too small to represent", + point=point, + ) + ) + continue + expected = req.numerator / milliseconds else: continue elif operation is DerivedOperation.THROUGHPUT_PER_PROVISIONED_KW: - denominator = artifacts.resolve(req["denominator"], check, point) + denominator = artifacts.resolve(req.denominator, check, point) if denominator is None or denominator <= 0: continue expected = ( @@ -178,14 +204,11 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul else None ) elif operation is DerivedOperation.OUTPUT_TOKENS_PER_COMPLETED_TURN_SECOND: - tokens = point.get_summary("output_tokens_per_turn_total") - seconds = point.get_summary("e2e_turn_time_seconds_total") - if ( - isinstance(tokens, (int, float)) - and isinstance(seconds, (int, float)) - and seconds > 0 - ): - expected = tokens / seconds + expected = ( + point.summary.e2e_avg_interactivity + if point.summary is not None + else None + ) if expected is None and stored is None: continue else: @@ -202,6 +225,11 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul result(check, False, "Metric inputs unavailable", point=point) ) continue + if not math.isfinite(expected): + findings.append( + result(check, False, "Derived metric is not finite", point=point) + ) + continue if stored is None: findings.append( result(check, True, f"Derived metric {expected}", point=point) diff --git a/src/inference_endpoint/validation/checks/power.py b/src/inference_endpoint/validation/checks/power.py index e3b5dd2c3..e56db9ac7 100644 --- a/src/inference_endpoint/validation/checks/power.py +++ b/src/inference_endpoint/validation/checks/power.py @@ -15,14 +15,17 @@ from ..artifacts import Artifacts, PointArtifacts, read_mapping from ..evaluator_base import Evaluator -from ..operations import PowerOperation +from ..operations import PowerMode, PowerOperation from ..planner import PlannedCheck, plan_checks from ..power.calculation import PowerCalculator from ..power.models import PowerComputation, PowerEvidence, SystemPower from ..results import CheckResult +from ..schemas.requirements_v1 import ( + PowerRequirements, +) from ..types import Cooling, Decision, EvidenceKey, Scope from ..vocabulary import CheckKind -from .helpers import finding, unsupported, validate_modes, warning +from .helpers import finding, unsupported, warning def power_entry(artifacts: Artifacts, point: PointArtifacts) -> PowerEvidence: @@ -64,6 +67,7 @@ def compute_descriptor(check: PlannedCheck, artifacts: Artifacts) -> None: """Compute only the selected system using its effective descriptor rule.""" system_id = check.subject.id rule = check.rule + req = check.requirements(PowerRequirements) existing = artifacts.derived.power.get(system_id, {}) if existing.get("requirements") == rule.requirements: return @@ -77,8 +81,9 @@ def compute_descriptor(check: PlannedCheck, artifacts: Artifacts) -> None: - {EvidenceKey.POWER_COMPUTATION, EvidenceKey.POINT_POWER} } ) - constants = artifacts.resolve(rule.requirements["constants"], check) - path = Path(system_id) / rule.requirements["artifact"] + assert req.artifact is not None + constants = artifacts.resolve(req.constants, check) + path = Path(system_id) / req.artifact if not path.exists(): artifacts.derived.power[system_id] = { "problems": [f"Power descriptor {path.name} is missing"] @@ -125,7 +130,7 @@ def compute_descriptor(check: PlannedCheck, artifacts: Artifacts) -> None: computation.sets[computation.sets.index(node_set)] = replace( node_set, accelerators_per_node=count ) - if rule.requirements.get("cooling_must_match_system_description"): + if req.cooling_must_match_system_description: declared = [str(p.get_system("cooling", "")) for p in members] declared.extend(str(n.get("cooling", "")) for n in nodes) labels: set[Cooling] = set() @@ -210,7 +215,7 @@ def prepare(artifacts: Artifacts) -> None: if ( check.decision is Decision.READY and check.rule.kind is CheckKind.POWER - and check.rule.requirements.get("operation") == operation + and check.requirements(PowerRequirements).operation == operation ): try: PowerEvaluator()(check, artifacts) @@ -237,22 +242,16 @@ def prepare(artifacts: Artifacts) -> None: class PowerEvaluator(Evaluator, kind=CheckKind.POWER): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - params = check.rule.requirements - invalid = validate_modes(check) - if invalid is not None: - return invalid - try: - operation = PowerOperation(cast(str, params.get("operation"))) - except ValueError: - return unsupported(check) + params = check.requirements(PowerRequirements) + operation = params.operation if operation is PowerOperation.DESCRIPTOR: flags = ( - "validate_public_sources", - "validate_node_sets", - "validate_scale_out", - "validate_computed_values", + params.validate_public_sources, + params.validate_node_sets, + params.validate_scale_out, + params.validate_computed_values, ) - if any(params.get(field) is False for field in flags): + if any(flag is False for flag in flags): return unsupported( check, "This power schema requires source, node, scale-out and computed-value validation", @@ -318,9 +317,9 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul } ) if ( - params.get("declared_total_scaling") != "maximum_engaged_fraction" - or params.get("switch_scaling") != "engaged_node_fraction" - or params.get("on_undeclared_nodes") != "fully_engaged" + params.declared_total_scaling != PowerMode.MAXIMUM_ENGAGED_FRACTION + or params.switch_scaling != PowerMode.ENGAGED_NODE_FRACTION + or params.on_undeclared_nodes != PowerMode.FULLY_ENGAGED ): return unsupported( check, "Unsupported engaged power scaling policy" @@ -345,7 +344,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul / sum(s.nodes_provisioned for s in computation.sets) ) kw = watts / 1000 - point.derived.power_kw = round(kw, params["round_kw_decimals"]) + point.derived.power_kw = round(kw, params.round_kw_decimals) point.context = point.context.model_copy( update={ "available": point.context.available @@ -361,9 +360,10 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul ) ) continue - factors = params.get( - "replica_factors", - ("tensor_parallel", "pipeline_parallel", "expert_parallel"), + factors = ( + params.replica_factors + if params.replica_factors is not None + else ("tensor_parallel", "pipeline_parallel", "expert_parallel") ) present = any(field in description for field in (*factors, "data_parallel")) replica = math.prod(description.get(field, 1) for field in factors) @@ -371,7 +371,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul known = all(s.accelerators_per_node is not None for s in computation.sets) if operation is PowerOperation.NODE_DECLARATION: if ( - params.get("require_accelerator_capacity") + params.require_accelerator_capacity and present and known and not description.get("disaggregated") @@ -430,9 +430,9 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul ) continue maximum = provisioned // replica - shortfall = artifacts.resolve(params["shortfall"], check, point) + shortfall = artifacts.resolve(params.shortfall, check, point) valid = dp <= maximum - if params.get("require_maximum_data_parallel") and dp < maximum: + if params.require_maximum_data_parallel and dp < maximum: valid = ( isinstance(shortfall, Mapping) and shortfall.get("dp_actual") == dp diff --git a/src/inference_endpoint/validation/checks/seeds.py b/src/inference_endpoint/validation/checks/seeds.py index 203e0b8e3..7c9f5f3a6 100644 --- a/src/inference_endpoint/validation/checks/seeds.py +++ b/src/inference_endpoint/validation/checks/seeds.py @@ -5,7 +5,6 @@ from __future__ import annotations from collections.abc import Mapping -from typing import cast from ..artifacts import Artifacts, nested from ..cohorts import Cohort, adoption_window @@ -13,22 +12,19 @@ from ..operations import SeedOperation from ..planner import PlannedCheck from ..results import CheckResult +from ..schemas.requirements_v1 import ( + SeedBindingRequirements, +) from ..vocabulary import CheckKind -from .helpers import finding, unsupported, validate_modes, warning +from .helpers import finding, unsupported, warning class SeedBindingEvaluator(Evaluator, kind=CheckKind.SEED_BINDING): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - params = check.rule.requirements - invalid = validate_modes(check) - if invalid is not None: - return invalid - try: - operation = SeedOperation(cast(str, params.get("operation"))) - except ValueError: - return unsupported(check) + params = check.requirements(SeedBindingRequirements) + operation = params.operation registry = ( - artifacts.resolve(params.get("catalog"), check) + artifacts.resolve(params.catalog, check) if operation is not SeedOperation.LEGACY_NAMES else None ) @@ -40,14 +36,15 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul for point in artifacts.members(check): runtime = point.get_config("runtime_settings.runtime", {}) if operation is SeedOperation.LEGACY_NAMES: - aliases = params["aliases"] + aliases = params.aliases + assert aliases is not None used = [ key for key in aliases if f"runtime_settings.runtime.{key}" in point.evidence.config.fields ] if used and ( - not params.get("warn_legacy_only_when_model_seed_missing") + not params.warn_legacy_only_when_model_seed_missing or runtime.get("model_seed") is None ): output.append( @@ -80,9 +77,10 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul ) continue if operation is SeedOperation.RUNTIME_VALUES: + assert params.fields is not None canonical = nested(point.config, "runtime_settings.runtime") problems = [] - for field in params["fields"]: + for field in params.fields: expected = nested(published, field) actual = ( nested(canonical, field) @@ -92,7 +90,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul if expected is None: return unsupported(check, f"Published seed set has no {field}") if actual != expected and ( - actual is not None or params.get("require_all") + actual is not None or params.require_all ): problems.append(f"{field}={actual!r}, expected {expected!r}") output.append( @@ -130,9 +128,8 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul ) ) continue - allowed = adoption_window( - publication, params["adoption_window_cohorts"] - ) + assert params.adoption_window_cohorts is not None + allowed = adoption_window(publication, params.adoption_window_cohorts) output.append( finding( check, diff --git a/src/inference_endpoint/validation/checks/drafters.py b/src/inference_endpoint/validation/checks/spec_decode_heads.py similarity index 50% rename from src/inference_endpoint/validation/checks/drafters.py rename to src/inference_endpoint/validation/checks/spec_decode_heads.py index a81f96e75..45d71ec4f 100644 --- a/src/inference_endpoint/validation/checks/drafters.py +++ b/src/inference_endpoint/validation/checks/spec_decode_heads.py @@ -1,42 +1,41 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Policy-driven drafters checks.""" +"""Policy-driven speculative decoding head checks.""" from __future__ import annotations from collections.abc import Mapping -from typing import cast from ..artifacts import Artifacts, nested from ..cohorts import Cohort from ..evaluator_base import Evaluator -from ..operations import DrafterOperation +from ..models import thaw +from ..operations import PolicyAction, SpecDecodeHeadOperation from ..planner import PlannedCheck from ..results import CheckResult +from ..schemas.requirements_v1 import ( + SpecDecodeHeadRequirements, +) from ..vocabulary import CheckKind -from .helpers import finding, unsupported, validate_modes, warning +from .helpers import finding, unsupported, warning -class DrafterBindingEvaluator(Evaluator, kind=CheckKind.DRAFTER_BINDING): +class SpecDecodeHeadEvaluator(Evaluator, kind=CheckKind.SPEC_DECODE_HEAD): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - params = check.rule.requirements - invalid = validate_modes(check) - if invalid is not None: - return invalid - try: - operation = DrafterOperation(cast(str, params.get("operation"))) - except ValueError: - return unsupported(check) - catalog = artifacts.resolve(params["catalog"], check) + params = check.requirements(SpecDecodeHeadRequirements) + operation = params.operation + catalog = artifacts.resolve(params.catalog, check) if catalog is None: - return unsupported(check, "Approved drafter catalog unavailable") + return unsupported( + check, "Approved speculative decoding head catalog unavailable" + ) output = [] for point in artifacts.members(check): - declared = point.get_config("speculative_decoding") - if declared is None: - declared = point.get_config("drafter") - if declared is None: - declared = {} + declared = ( + point.config.speculative_decoding if point.config is not None else None + ) + if declared is None and point.config is not None: + declared = point.config.spec_decode_head model = point.context.model_id matched = None entries = ( @@ -46,29 +45,37 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul if isinstance(entry, Mapping): if ( entry.get("model") == model - and entry.get("repository") == declared.get("repository") - and entry.get("revision") == declared.get("revision") + and declared is not None + and declared.matches( + entry.get("repository"), + entry.get("revision"), + target_checksum=entry.get("target_checksum"), + configuration=thaw(entry.get("configuration")), + ) ): matched = entry break else: return unsupported( check, - "Drafter catalog entries must disclose repository and revision for model identity matching", + "Speculative decoding head catalog entries must disclose a supported identity for model matching", ) - if operation is DrafterOperation.MEMBERSHIP: + if operation is SpecDecodeHeadOperation.MEMBERSHIP: output.append( finding( check, matched is not None, - "Drafter matches approved model, repository and revision" + "Speculative decoding head matches an approved identity for this model" if matched - else "Drafter does not match an approved identity", + else "Speculative decoding head does not match an approved identity", point=point, ) ) continue - if matched is None and params.get("on_unmatched_drafter") == "skip": + if ( + matched is None + and params.on_unmatched_spec_decode_head == PolicyAction.SKIP + ): continue approved = Cohort.parse(nested(matched, "approved_cohort") or "") target = Cohort.parse(point.get_config("target_cohort") or "") @@ -76,19 +83,20 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul output.append( warning( check, - "Drafter approval age cannot be verified without valid approval and target cohorts", + "Speculative decoding head approval age cannot be verified without valid approval and target cohorts", point, ) ) continue + assert params.minimum_cohorts is not None earliest = approved - for _ in range(params["minimum_cohorts"]): + for _ in range(params.minimum_cohorts): earliest = earliest.next() output.append( finding( check, target >= earliest, - f"Drafter target {target}; earliest permitted cohort {earliest}", + f"Speculative decoding head target {target}; earliest permitted cohort {earliest}", point=point, ) ) diff --git a/src/inference_endpoint/validation/checks/steady_state.py b/src/inference_endpoint/validation/checks/steady_state.py index fed99b0b2..029d8d674 100644 --- a/src/inference_endpoint/validation/checks/steady_state.py +++ b/src/inference_endpoint/validation/checks/steady_state.py @@ -5,7 +5,6 @@ from __future__ import annotations from collections.abc import Mapping -from typing import cast from ..artifacts import Artifacts from ..evaluator_base import Evaluator @@ -13,23 +12,20 @@ from ..operations import SteadyOperation from ..planner import PlannedCheck from ..results import CheckResult +from ..schemas.requirements_v1 import ( + SteadyStateRequirements, +) from ..vocabulary import CheckKind -from .helpers import finding, number, unsupported, validate_modes, warning +from .helpers import finding, number, unsupported, warning class SteadyStateEvaluator(Evaluator, kind=CheckKind.STEADY_STATE): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - params = check.rule.requirements - invalid = validate_modes(check) - if invalid is not None: - return invalid - constants = artifacts.resolve(params["constants"], check) + params = check.requirements(SteadyStateRequirements) + constants = artifacts.resolve(params.constants, check) if not isinstance(constants, Mapping): return unsupported(check, "Steady-state catalog unavailable") - try: - operation = SteadyOperation(cast(str, params.get("operation"))) - except ValueError: - return unsupported(check) + operation = params.operation output = [] for point in artifacts.members(check): block = point.config.steady_state if point.config is not None else None @@ -38,7 +34,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul output.append( warning( check, - f"Steady-state window is nonofficial; reporting basis: {params['fallback']}", + f"Steady-state window is nonofficial; reporting basis: {params.fallback}", point, ) ) @@ -98,7 +94,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul problems.append( f"Official window spans {spans} super-passes; requires {minimum}" ) - if params.get("official_requires_plateau") and any( + if params.official_requires_plateau and any( v is not MetricState.PLATEAU for v in block.state.values() ): problems.append("Official window contains a nonplateau metric") @@ -116,7 +112,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul else block.n_super_passes ) if ( - params.get("check_reported_super_pass_count") + params.check_reported_super_pass_count and reported is not None and spans is not None and reported != spans diff --git a/src/inference_endpoint/validation/checks/structure.py b/src/inference_endpoint/validation/checks/structure.py index 154d5c91a..d59fd5562 100644 --- a/src/inference_endpoint/validation/checks/structure.py +++ b/src/inference_endpoint/validation/checks/structure.py @@ -7,23 +7,38 @@ import re from collections.abc import Mapping from pathlib import Path -from typing import cast from inference_endpoint.config.schema import LoadPatternType from ..artifacts import Artifacts from ..evaluator_base import Evaluator +from ..operations import ( + ArtifactSchema as ArtifactSchemaKind, + PolicyAction, + PresenceObject, +) from ..outcomes import result from ..planner import PlannedCheck from ..results import CheckResult, Severity +from ..schemas.requirements_v1 import ( + ArtifactSchemaRequirements, + CohortIdentifierRequirements, + ConsistencyRequirements, + DisclosureRequirements, + IssuanceRequirements, + MembershipRequirements, + PathResolutionRequirements, + PresenceRequirements, + ReportRequirements, +) from ..vocabulary import CheckKind from .helpers import values class Presence(Evaluator, kind=CheckKind.PRESENCE): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements - targets = req.get("targets") or [req.get("target")] + req = check.requirements(PresenceRequirements) + targets = req.targets or [req.target] findings = [] for target in targets: value = ( @@ -32,21 +47,21 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul else artifacts.resolve(target, check) ) items = values(value) - if req.get("pattern"): + if req.pattern: items = [ item for item in items - if re.fullmatch(req["pattern"], getattr(item, "name", str(item))) + if re.fullmatch(req.pattern, getattr(item, "name", str(item))) ] - object_type = req.get("object") + object_type = req.object def present(item: object, object_type: str | None = object_type) -> bool: if isinstance(item, Path): return ( item.is_file() - if object_type == "file" + if object_type == PresenceObject.FILE else item.is_dir() - if object_type == "directory" + if object_type == PresenceObject.DIRECTORY else item.exists() ) return item is not None and item is not False @@ -55,8 +70,8 @@ def present(item: object, object_type: str | None = object_type) -> bool: findings.append( result( check, - count >= req.get("minimum_count", 1), - f"{target}: found {count}, required {req.get('minimum_count', 1)}", + count >= req.minimum_count, + f"{target}: found {count}, required {req.minimum_count}", ) ) return findings @@ -65,22 +80,22 @@ def present(item: object, object_type: str | None = object_type) -> bool: class ArtifactSchema(Evaluator, kind=CheckKind.ARTIFACT_SCHEMA): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: point = artifacts.index.points[check.subject.id] - req = check.rule.requirements - schema = req.get("schema") - if schema == "accuracy_result": + req = check.requirements(ArtifactSchemaRequirements) + schema = req.artifact_schema + if schema == ArtifactSchemaKind.ACCURACY_RESULT: present = point.evidence.accuracy.present - if req.get("when_present") and not present: + if req.when_present and not present: return [] obj = point.accuracy findings = point.evidence.accuracy.errors - if obj is not None and req.get("reject_empty") and not obj.root: + if obj is not None and req.reject_empty and not obj.root: return [result(check, False, "Accuracy results must not be empty")] else: attr = { - "point_config": "config", - "system_description": "system", - "result_summary": "summary", - }.get(cast(str, schema)) + ArtifactSchemaKind.POINT_CONFIG: "config", + ArtifactSchemaKind.SYSTEM_DESCRIPTION: "system", + ArtifactSchemaKind.RESULT_SUMMARY: "summary", + }.get(schema) if attr is None: return [ result( @@ -109,24 +124,24 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class Membership(Evaluator, kind=CheckKind.MEMBERSHIP): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements - catalog = artifacts.resolve(req["catalog"], check) - items = values(artifacts.resolve(req["target"], check)) + req = check.requirements(MembershipRequirements) + catalog = artifacts.resolve(req.catalog, check) + items = values(artifacts.resolve(req.target, check)) findings = [] seen = set() for value in items: - if value is None and req.get("skip_missing"): + if value is None and req.skip_missing: continue - if value is None and not req.get("required"): + if value is None and not req.required: continue - if req.get("deduplicate") and value in seen: + if req.deduplicate and value in seen: continue seen.add(value) findings.append( result( check, value in catalog if catalog is not None else False, - f"{req['target']}: {value!r}; allowed {list(catalog or [])!r}", + f"{req.target}: {value!r}; allowed {list(catalog or [])!r}", blocked=catalog is None, ) ) @@ -135,30 +150,30 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class Consistency(Evaluator, kind=CheckKind.CONSISTENCY): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements - if "left" in req: - left = artifacts.resolve(req["left"], check) - right = artifacts.resolve(req["right"], check) + req = check.requirements(ConsistencyRequirements) + if req.left is not None: + left = artifacts.resolve(req.left, check) + right = artifacts.resolve(req.right, check) if left is None or right is None: finding = result(check, False, "Comparison evidence is missing") - if req.get("on_missing") == "warning": + if req.on_missing == PolicyAction.WARNING: finding = finding.model_copy(update={"severity": Severity.WARNING}) return [finding] return [result(check, left == right, f"{left!r} compared with {right!r}")] - items = values(artifacts.resolve(req["target"], check)) - if req.get("exclude_fields"): + items = values(artifacts.resolve(req.target, check)) + if req.exclude_fields: items = [ - {k: v for k, v in item.items() if k not in req["exclude_fields"]} + {k: v for k, v in item.items() if k not in req.exclude_fields} if isinstance(item, Mapping) else item for item in items ] if any(item is None for item in items): - if req.get("on_missing") == "error": + if req.on_missing == PolicyAction.ERROR: return [result(check, False, "Required comparison evidence is missing")] items = [item for item in items if item is not None] nonempty = ( - not req.get("require_nonempty") + not req.require_nonempty or bool(items) and all(bool(item) for item in items) ) @@ -166,16 +181,16 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul result( check, nonempty and (not items or all(item == items[0] for item in items)), - f"{req['target']}: {len(items)} declarations compared", + f"{req.target}: {len(items)} declarations compared", ) ] class Disclosure(Evaluator, kind=CheckKind.DISCLOSURE): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements + req = check.requirements(DisclosureRequirements) point = artifacts.index.points[check.subject.id] - fields = artifacts.resolve(req["fields"], check) + fields = artifacts.resolve(req.fields, check) missing = [name for name in fields if point.get_config(name) in (None, "")] return [ result( @@ -190,7 +205,9 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class CohortIdentifier(Evaluator, kind=CheckKind.COHORT_IDENTIFIER): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - value = artifacts.resolve(check.rule.requirements["target"], check) + value = artifacts.resolve( + check.requirements(CohortIdentifierRequirements).target, check + ) return [ result( check, @@ -203,12 +220,12 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class PathResolution(Evaluator, kind=CheckKind.PATH_RESOLUTION): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - req = check.rule.requirements - root = Path(artifacts.resolve(req["base"], check)).resolve() + req = check.requirements(PathResolutionRequirements) + root = Path(artifacts.resolve(req.base, check)).resolve() findings = [] - for address in req["targets"]: + for address in req.targets: value = artifacts.resolve(address, check) - if value is None and req.get("on_missing") == "skip": + if value is None and req.on_missing == PolicyAction.SKIP: continue if not isinstance(value, str): findings.append(result(check, False, f"{address}: missing path")) @@ -224,17 +241,24 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul ) continue valid = ( - (req.get("allow_absolute", False) or not path.is_absolute()) - and (req.get("allow_parent_traversal", False) or ".." not in path.parts) + (req.allow_absolute or not path.is_absolute()) and ( - req.get("allow_escape_via_symlink", False) - or resolved.is_relative_to(root) + ( + req.allow_parent_traversal + if req.allow_parent_traversal is not None + else False + ) + or ".." not in path.parts ) and ( - resolved.is_dir() - if req.get("require_directory") - else resolved.exists() + ( + req.allow_escape_via_symlink + if req.allow_escape_via_symlink is not None + else False + ) + or resolved.is_relative_to(root) ) + and (resolved.is_dir() if req.require_directory else resolved.exists()) ) findings.append( result(check, valid, f"{address}: {value!r} resolves to {resolved}") @@ -244,19 +268,25 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul class ReportValue(Evaluator, kind=CheckKind.REPORT): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - value = artifacts.resolve(check.rule.requirements["target"], check) - return [result(check, True, f"{check.rule.requirements['target']}: {value!r}")] + value = artifacts.resolve(check.requirements(ReportRequirements).target, check) + return [ + result( + check, + True, + f"{check.requirements(ReportRequirements).target}: {value!r}", + ) + ] class Issuance(Evaluator, kind=CheckKind.ISSUANCE): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: point = artifacts.index.points[check.subject.id] - req = check.rule.requirements + req = check.requirements(IssuanceRequirements) pattern = point.get_config( "runtime_settings.load_pattern", LoadPatternType.CONCURRENCY ) - valid = pattern in req["allowed"] and ( - not req.get("require_positive_concurrency") + valid = pattern in req.allowed and ( + not req.require_positive_concurrency or point.concurrency is not None and point.concurrency > 0 ) diff --git a/src/inference_endpoint/validation/checks/warmup.py b/src/inference_endpoint/validation/checks/warmup.py index b294b5b79..1d8968429 100644 --- a/src/inference_endpoint/validation/checks/warmup.py +++ b/src/inference_endpoint/validation/checks/warmup.py @@ -5,32 +5,28 @@ from __future__ import annotations from collections.abc import Mapping -from typing import cast from ..artifacts import Artifacts from ..evaluator_base import Evaluator -from ..operations import WarmupOperation +from ..operations import PolicyAction, WarmupOperation from ..planner import PlannedCheck from ..results import CheckResult +from ..schemas.requirements_v1 import ( + WarmupRequirements, +) from ..vocabulary import CheckKind -from .helpers import block_value, finding, unsupported, validate_modes +from .helpers import block_value, finding, unsupported class WarmupEvaluator(Evaluator, kind=CheckKind.WARMUP): def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResult]: - params = check.rule.requirements - invalid = validate_modes(check) - if invalid is not None: - return invalid - try: - operation = WarmupOperation(cast(str, params.get("operation"))) - except ValueError: - return unsupported(check) + params = check.requirements(WarmupRequirements) + operation = params.operation output = [] for point in artifacts.members(check): if operation is WarmupOperation.SALT_DISABLED: - value = artifacts.resolve(params["source"], check, point) - if value is None and params.get("on_missing") == "skip": + value = artifacts.resolve(params.source, check, point) + if value is None and params.on_missing == PolicyAction.SKIP: continue output.append( finding( @@ -41,7 +37,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul ) ) continue - if params.get("verify_archive_contents"): + if params.verify_archive_contents: return unsupported( check, "Warmup archive content verification is unavailable" ) @@ -61,7 +57,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul warmup.get(key) == 0 for key in ("duration_s", "requests_issued", "requests_completed") ) - if disabled and params.get("disabled_warmup") == "exempt": + if disabled and params.disabled_warmup == PolicyAction.EXEMPT: output.append( finding( check, @@ -75,7 +71,7 @@ def __call__(self, check: PlannedCheck, artifacts: Artifacts) -> list[CheckResul output.append( finding( check, - retained or not params.get("require_declared_retained", True), + retained or not params.require_declared_retained, "Warmup logs retained" if retained else "Warmup log retention is not declared", diff --git a/src/inference_endpoint/validation/evidence/loaders.py b/src/inference_endpoint/validation/evidence/loaders.py index a29b47ab6..8655cc4bd 100644 --- a/src/inference_endpoint/validation/evidence/loaders.py +++ b/src/inference_endpoint/validation/evidence/loaders.py @@ -18,14 +18,41 @@ from .point_summary import PointSummary from .system import SystemDescription +MAX_ARTIFACT_DEPTH = 100 + + +class ArtifactStructureError(ValueError): + """An artifact contains a cyclic or excessively nested structure.""" + def supplied_fields(value: object, prefix: str = "") -> frozenset[str]: - fields = set() - if isinstance(value, Mapping): - for key, child in value.items(): - name = f"{prefix}.{key}" if prefix else str(key) - fields.add(name) - fields.update(supplied_fields(child, name)) + fields: set[str] = set() + ancestors: set[int] = set() + stack: list[tuple[object, str, int, bool]] = [(value, prefix, 0, False)] + while stack: + current, address, depth, leaving = stack.pop() + if not isinstance(current, (Mapping, list, tuple)): + continue + identity = id(current) + if leaving: + ancestors.remove(identity) + continue + if identity in ancestors: + raise ArtifactStructureError("Artifact contains a recursive alias") + if depth > MAX_ARTIFACT_DEPTH: + raise ArtifactStructureError( + f"Artifact nesting exceeds {MAX_ARTIFACT_DEPTH} levels" + ) + ancestors.add(identity) + stack.append((current, address, depth, True)) + if isinstance(current, Mapping): + for key, child in current.items(): + name = f"{address}.{key}" if address else str(key) + fields.add(name) + stack.append((child, name, depth + 1, False)) + else: + for child in current: + stack.append((child, address, depth + 1, False)) return frozenset(fields) @@ -86,7 +113,13 @@ class ParsedArtifact(Generic[T]): # noqa: UP046 - supported by the repository t def from_json( cls, data: object, model: type[T], path: Path, rule: str ) -> "ParsedArtifact[T]": - fields = supplied_fields(data) + try: + fields = supplied_fields(data) + except ArtifactStructureError as error: + finding = CheckResult( + rule=rule, key="artifact-structure", message=str(error), path=path + ) + return cls(None, frozenset(), (finding,), True) try: return cls(model.model_validate(data), fields, (), True) except ValidationError as error: @@ -110,7 +143,13 @@ def read_artifact( # noqa: UP047 - supported by the repository type checker if path.suffix in {".yaml", ".yml"} else json.loads(text) ) - except (OSError, UnicodeError, json.JSONDecodeError, yaml.YAMLError) as error: + except ( + OSError, + UnicodeError, + json.JSONDecodeError, + yaml.YAMLError, + RecursionError, + ) as error: if accuracy_scores: # Result-file checks own failures in the inline accuracy source. return ParsedArtifact(None, frozenset(), (), False) diff --git a/src/inference_endpoint/validation/evidence/point_config.py b/src/inference_endpoint/validation/evidence/point_config.py index 6cd20a452..e1c248b53 100644 --- a/src/inference_endpoint/validation/evidence/point_config.py +++ b/src/inference_endpoint/validation/evidence/point_config.py @@ -5,18 +5,23 @@ from __future__ import annotations from pydantic import ( + AliasChoices, BaseModel, ConfigDict, Field, JsonValue, StrictBool, StrictInt, + field_serializer, + field_validator, model_validator, ) from inference_endpoint.config.schema import LoadPatternType -from ..types import Division, OfflineMode +from ..json_values import json_identity_equal +from ..operations import ChecksumAlgorithm +from ..types import ChecksumValue, CommitId, Division, OfflineMode from .steady_state import SteadyState @@ -77,8 +82,85 @@ class CheckpointDeclaration(BaseModel): revision: str | None = None -class DecodeHeadDeclaration(CheckpointDeclaration): - weight_checksum: str | None = None +class SpecDecodeHeadChecksum(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + algorithm: ChecksumAlgorithm + revision: CommitId + + @classmethod + def from_string(cls, value: str) -> SpecDecodeHeadChecksum: + algorithm, separator, revision = value.partition(":") + if not separator: + raise ValueError("Weight checksum requires algorithm:revision") + return cls.model_validate({"algorithm": algorithm, "revision": revision}) + + def as_string(self) -> str: + return f"{self.algorithm.value}:{self.revision}" + + +class SpecDecodeHeadDeclaration(CheckpointDeclaration): + weight_checksum: SpecDecodeHeadChecksum | None = None + target_checksum: ChecksumValue | None = None + configuration: dict[str, JsonValue] | None = None + + @field_validator("weight_checksum", mode="before") + @classmethod + def parse_checksum(cls, value: object) -> SpecDecodeHeadChecksum | None: + if value is None or isinstance(value, SpecDecodeHeadChecksum): + return value + if not isinstance(value, str): + raise ValueError("Weight checksum must be an algorithm:revision string") + return SpecDecodeHeadChecksum.from_string(value) + + @field_serializer("weight_checksum") + def serialize_checksum(self, value: SpecDecodeHeadChecksum | None) -> str | None: + return value.as_string() if value is not None else None + + @model_validator(mode="after") + def consistent_identity(self) -> SpecDecodeHeadDeclaration: + if self.target_checksum is not None or self.configuration is not None: + if self.target_checksum is None or not self.configuration: + raise ValueError( + "Configuration identity requires target_checksum and nonempty configuration" + ) + if any( + value is not None + for value in (self.weight_checksum, self.repository, self.revision) + ): + raise ValueError( + "Weight and configuration identity forms cannot be combined" + ) + if ( + self.weight_checksum is not None + and self.revision is not None + and self.revision != self.weight_checksum.revision + ): + raise ValueError("Weight checksum and revision must agree") + return self + + def matches( + self, + repository: str | None = None, + revision: str | None = None, + *, + target_checksum: str | None = None, + configuration: JsonValue = None, + ) -> bool: + if target_checksum is not None: + return ( + self.target_checksum == target_checksum + and bool(self.configuration) + and json_identity_equal(self.configuration, configuration) + ) + if self.target_checksum is not None: + return False + if repository is None or revision is None: + return False + if self.weight_checksum is not None: + return self.weight_checksum.revision == revision and ( + self.repository is None or self.repository == repository + ) + return self.repository == repository and self.revision == revision class AgenticSettings(BaseModel): @@ -154,13 +236,28 @@ class PointConfig(BaseModel): dataset_link: str | None = None offline: OfflineMode | None = None steady_state: SteadyState | None = None - speculative_decoding: DecodeHeadDeclaration | None = None - drafter: DecodeHeadDeclaration | None = None + speculative_decoding: SpecDecodeHeadDeclaration | None = None + spec_decode_head: SpecDecodeHeadDeclaration | None = Field( + default=None, validation_alias=AliasChoices("spec_decode_head", "drafter") + ) checkpoint: CheckpointDeclaration | None = None accuracy: dict[str, dict[str, JsonValue]] | None = None nodes_used: list[NodesUsed] | None = None dp_shortfall: DpShortfall | None = None shared_src: str | None = None + + @model_validator(mode="before") + @classmethod + def validate_spec_decode_head_aliases(cls, value: object) -> object: + if ( + isinstance(value, dict) + and "spec_decode_head" in value + and "drafter" in value + and not json_identity_equal(value["spec_decode_head"], value["drafter"]) + ): + raise ValueError("spec_decode_head and its input alias must agree") + return value + shared_docs: str | None = None seed_set: str | None = None target_cohort: str | None = None diff --git a/src/inference_endpoint/validation/evidence/point_summary.py b/src/inference_endpoint/validation/evidence/point_summary.py index e104477b4..735e30ba4 100644 --- a/src/inference_endpoint/validation/evidence/point_summary.py +++ b/src/inference_endpoint/validation/evidence/point_summary.py @@ -18,8 +18,8 @@ class PercentileStats(BaseModel): """Summary statistics dict produced by the endpoints ``compute_summary()`` helper.""" model_config = ConfigDict(extra="ignore") - total: float = 0.0 - percentiles: dict[str, float] = Field(default_factory=dict) + total: FiniteFloat = 0.0 + percentiles: dict[str, FiniteFloat] = Field(default_factory=dict) @field_validator("percentiles") @classmethod @@ -69,12 +69,13 @@ class PointSummary(BaseModel): n_samples_issued: int = 0 n_samples_completed: int n_samples_failed: int = 0 - duration_ns: float + duration_ns: FiniteFloat + latency: PercentileStats = Field(default_factory=PercentileStats) ttft: PercentileStats = Field(default_factory=PercentileStats) tpot: PercentileStats = Field(default_factory=PercentileStats) output_sequence_lengths: PercentileStats = Field(default_factory=PercentileStats) - output_tokens_per_turn_total: float | None = None - e2e_turn_time_seconds_total: float | None = None + output_tokens_per_turn_total: FiniteFloat | None = None + e2e_turn_time_seconds_total: FiniteFloat | None = None @computed_field # type: ignore[misc] @property @@ -138,6 +139,9 @@ def e2e_avg_interactivity(self) -> float | None: """§4.1: ``sum(output_tokens_per_turn) / sum(e2e_turn_time_seconds)``.""" tokens = self.output_tokens_per_turn_total seconds = self.e2e_turn_time_seconds_total + if tokens is None and seconds is None and self.n_samples_failed == 0: + tokens = self.output_sequence_lengths.total + seconds = self.latency.total / 1e9 if tokens is None or seconds is None or seconds <= 0: return None return tokens / seconds diff --git a/src/inference_endpoint/validation/json_values.py b/src/inference_endpoint/validation/json_values.py new file mode 100644 index 000000000..8ae179295 --- /dev/null +++ b/src/inference_endpoint/validation/json_values.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Type-preserving equality for JSON approval identities.""" + +import math + +from pydantic import JsonValue + + +def json_identity_equal(left: JsonValue, right: JsonValue) -> bool: + """Compare JSON structures without coercing scalar types or matching nonfinite values.""" + if type(left) is not type(right): + return False + if isinstance(left, dict) and isinstance(right, dict): + return left.keys() == right.keys() and all( + json_identity_equal(value, right[key]) for key, value in left.items() + ) + if isinstance(left, list) and isinstance(right, list): + return len(left) == len(right) and all( + json_identity_equal(a, b) for a, b in zip(left, right, strict=True) + ) + if isinstance(left, float) and isinstance(right, float): + return math.isfinite(left) and math.isfinite(right) and left == right + return left == right diff --git a/src/inference_endpoint/validation/models.py b/src/inference_endpoint/validation/models.py index c601a5218..5eefec338 100644 --- a/src/inference_endpoint/validation/models.py +++ b/src/inference_endpoint/validation/models.py @@ -8,6 +8,7 @@ from typing import Any, TypeVar, overload from .conditions import Conditions +from .requirements import CheckRequirements from .types import EvidenceKey, PolicyFile, Release, Scope, Severity from .vocabulary import CheckKind, EvidenceReference @@ -49,7 +50,7 @@ class Rule: applies_to: Conditions | None unless: Conditions | None requires: frozenset[EvidenceKey] - requirements: Mapping[str, Any] + requirements: CheckRequirements references: frozenset[EvidenceReference] source: PolicyFile needs_model: bool = False diff --git a/src/inference_endpoint/validation/operations.py b/src/inference_endpoint/validation/operations.py index 144919e2b..0232df5d3 100644 --- a/src/inference_endpoint/validation/operations.py +++ b/src/inference_endpoint/validation/operations.py @@ -8,6 +8,7 @@ class BindingMatch(StrEnum): CHECKPOINT = "model_and_repository_and_revision" CLIENT_ALLOWLIST = "exact_revision" + SPEC_DECODE_HEAD = "model_and_identity" class AccuracyOperation(StrEnum): @@ -29,7 +30,7 @@ class OfflineOperation(StrEnum): ORDERING = "ordering" -class DrafterOperation(StrEnum): +class SpecDecodeHeadOperation(StrEnum): MEMBERSHIP = "membership" APPROVAL_AGE = "approval_age" @@ -43,6 +44,7 @@ class ComparisonOperator(StrEnum): class OperandKind(StrEnum): FIELD = "field" CONSTANT = "constant" + SUM = "sum" class DerivedOperation(StrEnum): @@ -88,3 +90,113 @@ class CountOperator(StrEnum): GREATER_THAN_OR_EQUAL = "greater_than_or_equal" EQUAL = "equal" POSITIVE_MULTIPLE = "positive_multiple" + + +class FractionConversion(StrEnum): + LEGACY_VALUE_AND_THRESHOLD_HEURISTIC = "legacy_value_and_threshold_heuristic" + + +class AccuracyMode(StrEnum): + ARITHMETIC_MEAN = "arithmetic_mean" + CASE_INSENSITIVE = "case_insensitive" + PER_REQUIRED_DATASET_OR_SUITE = "per_required_dataset_or_suite" + SAMPLE_WEIGHTED_MEAN = "sample_weighted_mean" + SAMPLES_TIMES_REPEATS = "samples_times_repeats" + SCORE_OR_FIRST_METRIC = "score_or_first_metric" + SOLE_METRIC_ONLY_WITH_WARNING = "sole_metric_only_with_warning" + + +class ArtifactPrecedence(StrEnum): + STANDALONE_FIRST = "standalone_first" + + +class ArtifactSchema(StrEnum): + ACCURACY_RESULT = "accuracy_result" + APPENDIX_E_SYSTEM_POWER = "appendix_e_system_power" + POINT_CONFIG = "point_config" + RESULT_SUMMARY = "result_summary" + SYSTEM_DESCRIPTION = "system_description" + + +class ConsistencyMode(StrEnum): + EXACT = "exact" + WHOLE_DOCUMENT = "whole_document" + + +class CurveTypePolicy(StrEnum): + SINGLE_TURN = "single_turn" + + +class Deduplication(StrEnum): + DECLARED_NAME = "declared_name" + + +class IdentifierFormat(StrEnum): + GIT_COMMIT_SHA1 = "git_commit_sha1" + YEAR_MONTH_C0_OR_C1 = "year_month_c0_or_c1" + + +class PolicyAction(StrEnum): + AVERAGE_AND_WARN = "average_and_warn" + BLOCK_DEPENDENT_CHECKS = "block_dependent_checks" + BLOCKED = "blocked" + ERROR = "error" + EXCLUDE = "exclude" + EXEMPT = "exempt" + GATE_AVAILABLE_MEAN_AND_WARN = "gate_available_mean_and_warn" + REPORT_DERIVED = "report_derived" + SKIP = "skip" + SKIP_COUNT_COMPARISON = "skip_count_comparison" + WARN_RANGE_OR_SLOPE_REQUIRED = "warn_range_or_slope_required" + WARNING = "warning" + + +class PowerMode(StrEnum): + ENGAGED_NODE_FRACTION = "engaged_node_fraction" + FULLY_ENGAGED = "fully_engaged" + MAXIMUM_ENGAGED_FRACTION = "maximum_engaged_fraction" + + +class PowerTag(StrEnum): + ESTIMATED_POWER = "estimated_power" + + +class PresenceObject(StrEnum): + DECLARATION = "declaration" + DIRECTORY = "directory" + FILE = "file" + PATH = "path" + + +class RegionSource(StrEnum): + C_MAX_POINT = "c_max_point" + POINTS_WITH_READABLE_THROUGHPUT_AND_UTILIZATION = ( + "points_with_readable_throughput_and_utilization" + ) + SMALLEST_SUBMITTED = "smallest_submitted" + + +class ReportInterpretation(StrEnum): + CLIENT_IPC_FORWARDING_ONLY = "client_ipc_forwarding_only" + + +class ReportOperation(StrEnum): + BLOCKED_CONFIG_DEPENDENTS = "blocked_config_dependents" + + +class ReportingBasis(StrEnum): + WHOLE_RUN_TOTAL = "whole_run_total" + + +class ChecksumAlgorithm(StrEnum): + GIT_SHA1 = "git-sha1" + + +class AccuracyPresenceSource(StrEnum): + STANDALONE_ACCURACY_RESULTS = "standalone_accuracy_results" + EMBEDDED_NONEMPTY_ACCURACY_SCORES = "embedded_nonempty_accuracy_scores" + + +class AccuracyArtifactSource(StrEnum): + ACCURACY_RESULTS = "accuracy_results.json" + INLINE_ACCURACY_SCORES = "results.json.accuracy_scores" diff --git a/src/inference_endpoint/validation/planner.py b/src/inference_endpoint/validation/planner.py index 11342840e..52c4b5fc7 100644 --- a/src/inference_endpoint/validation/planner.py +++ b/src/inference_endpoint/validation/planner.py @@ -4,13 +4,16 @@ from collections.abc import Iterable from dataclasses import dataclass, replace -from types import MappingProxyType +from typing import TypeVar from .conditions import Context from .models import Override, Policy, Rule from .references import evidence_references, needs_model +from .requirements import CheckRequirements from .types import Decision, EvidenceKey, Match +_Requirements = TypeVar("_Requirements", bound=CheckRequirements) + @dataclass(frozen=True) class PlannedCheck: @@ -22,6 +25,12 @@ class PlannedCheck: override: str | None = None selected_members: tuple[str, ...] | None = None + def requirements(self, contract: type[_Requirements]) -> _Requirements: + parameters = self.rule.requirements + if not isinstance(parameters, contract): + raise TypeError(f"Rule {self.rule.id} requires {contract.__name__}") + return parameters + @dataclass(frozen=True) class CheckPlan: @@ -164,12 +173,14 @@ def plan_checks(policy: Policy, subjects: Iterable[Context]) -> CheckPlan: if not override.enabled: decision, reason = Decision.EXCLUDED, override.reason else: - requirements = {**rule.requirements, **override.requirements} + requirements = rule.requirements.with_updates( + override.requirements + ) rule = replace( rule, - requirements=MappingProxyType(requirements), - references=evidence_references(requirements), - needs_model=needs_model(requirements), + requirements=requirements, + references=evidence_references(requirements.wire()), + needs_model=needs_model(requirements.wire()), ) reason = override.reason if decision is Decision.READY: diff --git a/src/inference_endpoint/validation/policies/2026-10-C1/catalog.yaml b/src/inference_endpoint/validation/policies/2026-10-C1/catalog.yaml index c148ec8cc..aa0b3423c 100644 --- a/src/inference_endpoint/validation/policies/2026-10-C1/catalog.yaml +++ b/src/inference_endpoint/validation/policies/2026-10-C1/catalog.yaml @@ -129,33 +129,39 @@ catalogs: minimum_multiplier: 0.9, maximum_multiplier: 1.1, } - approved_sped_decode_heads: + approved_spec_decode_heads: - model: kimi-k3 repository: RadixArk/Kimi-K3-DSpark revision: 3c5bac301d9cf392706189d82ed947feca6c2f0f + approved_cohort: 2026-09-C1 method: dspark - model: kimi-k3 repository: Inferact/Kimi-K3-DSpark revision: cf6b8244620e7ea4b0651d214f28e89eac75bed6 + approved_cohort: 2026-09-C1 method: dspark - model: deepseek-v4_1-flash repository: deepseek-ai/DeepSeek-V4.1-Flash revision: dba1be0a40aa45a94ad051997016db3960a90277 + approved_cohort: 2026-09-C1 method: dspark bundled: true - model: qwen3_6-35b-a3b repository: Qwen/Qwen3.6-35B-A3B revision: 995ad96eacd98c81ed38be0c5b274b04031597b0 + approved_cohort: 2026-09-C1 method: mtp bundled: true - model: qwen3_6-35b-a3b repository: Qwen/Qwen3.6-35B-A3B-FP8 revision: 95a723d08a9490559dae23d0cff1d9466213d989 + approved_cohort: 2026-09-C1 method: mtp bundled: true - model: qwen3_6-35b-a3b repository: nvidia/Qwen3.6-35B-A3B-NVFP4 revision: 1355db6a052410cfd62085d94b58866fd0f2c3c5 + approved_cohort: 2026-09-C1 method: mtp bundled: true approved_client_revisions: [] @@ -313,9 +319,9 @@ diagnostics: catalog: context.cohorts.seed_sets, on_unavailable: block_dependent_checks, } - drafter-list-registry: + spec-decode-head-catalog: { kind: catalog_integrity, - catalog: catalogs.approved_sped_decode_heads, + catalog: catalogs.approved_spec_decode_heads, on_unavailable: block_dependent_checks, } diff --git a/src/inference_endpoint/validation/policies/2026-10-C1/point_checks.yaml b/src/inference_endpoint/validation/policies/2026-10-C1/point_checks.yaml index add565a2c..9830c708c 100644 --- a/src/inference_endpoint/validation/policies/2026-10-C1/point_checks.yaml +++ b/src/inference_endpoint/validation/policies/2026-10-C1/point_checks.yaml @@ -221,7 +221,11 @@ checks: metric-consistency-accounting: kind: comparison when: { field: result_summary.n_samples_issued, greater_than: 0 } - left: { kind: field, field: result_summary.n_samples_completed } + left: + kind: sum + operands: + - { kind: field, field: result_summary.n_samples_completed } + - { kind: field, field: result_summary.n_samples_failed, default: 0 } operator: equal right: { kind: field, field: result_summary.n_samples_issued } metric-consistency-output-tokens: @@ -299,23 +303,23 @@ checks: adoption_window_cohorts: 4 on_legacy_registry_without_cohorts: blocked requires: [point_config, seed_catalog] - approved-drafter: - kind: drafter_binding + approved-spec-decode-head: + kind: spec_decode_head operation: membership applies_to: { speculative_decoding: true } - requires: [approved_sped_decode_heads, point_config] - catalog: catalogs.approved_sped_decode_heads - matching: model_and_repository_and_revision - on_missing_repository_or_revision: error + requires: [approved_spec_decode_heads, point_config] + catalog: catalogs.approved_spec_decode_heads + matching: model_and_identity + on_missing_identity: error on_empty_catalog: error - drafter-approval-lead-time: - kind: drafter_binding + spec-decode-head-approval-lead-time: + kind: spec_decode_head operation: approval_age applies_to: { speculative_decoding: true } - requires: [approved_sped_decode_heads, point_config] - catalog: catalogs.approved_sped_decode_heads + requires: [approved_spec_decode_heads, point_config] + catalog: catalogs.approved_spec_decode_heads minimum_cohorts: 2 - on_unmatched_drafter: skip + on_unmatched_spec_decode_head: skip on_unparseable_cohort: warning on_missing_approval_cohort: warning nodes-used: diff --git a/src/inference_endpoint/validation/policies/2026-10-C1/submission_checks.yaml b/src/inference_endpoint/validation/policies/2026-10-C1/submission_checks.yaml index 29f2e9b8d..d55bbb785 100644 --- a/src/inference_endpoint/validation/policies/2026-10-C1/submission_checks.yaml +++ b/src/inference_endpoint/validation/policies/2026-10-C1/submission_checks.yaml @@ -27,10 +27,3 @@ checks: minimum_models: 1, accept: [standalone_accuracy_results, embedded_nonempty_accuracy_scores], } - system-name-consistency: - kind: consistency - target: point.system_description.system_name - comparison: exact - require_nonempty: true - required_artifact: point.system_desc.json - on_missing: error diff --git a/src/inference_endpoint/validation/policies/2026-10-C1/system_checks.yaml b/src/inference_endpoint/validation/policies/2026-10-C1/system_checks.yaml index 718b314d8..72ec02138 100644 --- a/src/inference_endpoint/validation/policies/2026-10-C1/system_checks.yaml +++ b/src/inference_endpoint/validation/policies/2026-10-C1/system_checks.yaml @@ -44,3 +44,10 @@ checks: config_summary_notes, ] comparison: whole_document + system-name-consistency: + kind: consistency + target: point.system_description.system_name + comparison: exact + require_nonempty: true + required_artifact: point.system_desc.json + on_missing: error diff --git a/src/inference_endpoint/validation/power/models.py b/src/inference_endpoint/validation/power/models.py index e40ad067a..cb807d488 100644 --- a/src/inference_endpoint/validation/power/models.py +++ b/src/inference_endpoint/validation/power/models.py @@ -4,12 +4,12 @@ from __future__ import annotations -from collections.abc import Mapping from dataclasses import dataclass, field from typing import Literal, TypedDict from pydantic import BaseModel, ConfigDict, Field, model_validator +from ..requirements import CheckRequirements from ..types import Cooling _W_PER_KW = 1000.0 @@ -213,4 +213,4 @@ class PowerEvidence(TypedDict, total=False): computation: PowerComputation descriptor: SystemPower problems: list[str] - requirements: Mapping[str, object] + requirements: CheckRequirements diff --git a/src/inference_endpoint/validation/references.py b/src/inference_endpoint/validation/references.py index 5875c7ff0..3a9c7eee8 100644 --- a/src/inference_endpoint/validation/references.py +++ b/src/inference_endpoint/validation/references.py @@ -5,6 +5,7 @@ import re from collections.abc import Mapping +from .operations import BindingMatch from .vocabulary import EvidenceReference @@ -52,5 +53,6 @@ def needs_model(requirements: Mapping) -> bool: ) or requirements.get("models") == "catalogs.models" or "constraints_by_model" in requirements - or requirements.get("matching") == "model_and_repository_and_revision" + or requirements.get("matching") + in {BindingMatch.CHECKPOINT, BindingMatch.SPEC_DECODE_HEAD} ) diff --git a/src/inference_endpoint/validation/requirements.py b/src/inference_endpoint/validation/requirements.py new file mode 100644 index 000000000..18c781778 --- /dev/null +++ b/src/inference_endpoint/validation/requirements.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Typed immutable check parameters retained across parsing and planning.""" + +from collections.abc import Mapping +from typing import Self + +from .types import FrozenModel + + +class CheckRequirements(FrozenModel): + def wire(self) -> dict[str, object]: + return self.model_dump(mode="python", by_alias=True, exclude_unset=True) + + def with_updates(self, updates: Mapping[str, object]) -> Self: + return type(self).model_validate({**self.wire(), **updates}) diff --git a/src/inference_endpoint/validation/schemas/requirements_v1.py b/src/inference_endpoint/validation/schemas/requirements_v1.py index 532edaf22..6e18b7446 100644 --- a/src/inference_endpoint/validation/schemas/requirements_v1.py +++ b/src/inference_endpoint/validation/schemas/requirements_v1.py @@ -3,29 +3,56 @@ """Strict check requirement contracts for cohort revision 1.""" from collections.abc import Mapping +from types import MappingProxyType from typing import Annotated, Literal, Self -from pydantic import Field, StrictBool, StrictFloat, StrictInt, model_validator +from pydantic import ( + Field, + StrictBool, + StrictFloat, + StrictInt, + field_serializer, + model_validator, +) from inference_endpoint.config.schema import LoadPatternType from ..operations import ( + AccuracyArtifactSource, + AccuracyMode, AccuracyOperation, + AccuracyPresenceSource, + ArtifactPrecedence, + ArtifactSchema, BindingMatch, ComparisonOperator, + ConsistencyMode, CountOperator, + CurveTypePolicy, + Deduplication, DerivedOperation, - DrafterOperation, DurationBasis, + FractionConversion, + IdentifierFormat, OfflineOperation, OperandKind, + PolicyAction, + PowerMode, PowerOperation, + PowerTag, + PresenceObject, RegionPlacementOperation, + RegionSource, + ReportingBasis, + ReportInterpretation, + ReportOperation, SeedOperation, + SpecDecodeHeadOperation, SteadyOperation, WarmupOperation, ) -from ..types import AccuracyKind, FrozenModel, Identifier, OfflineMode, SampleUnit +from ..requirements import CheckRequirements +from ..types import AccuracyKind, Identifier, OfflineMode, SampleUnit from ..vocabulary import CheckKind Number = Annotated[StrictInt | StrictFloat, Field(allow_inf_nan=False)] @@ -35,7 +62,7 @@ PositiveInt = Annotated[StrictInt, Field(gt=0)] -class Requirements(FrozenModel): +class Requirements(CheckRequirements): @model_validator(mode="before") @classmethod def reject_null_fields(cls, value: object) -> object: @@ -95,6 +122,7 @@ class ComparisonWhen(Requirements): class FieldOperand(Requirements): kind: Literal[OperandKind.FIELD] field: Identifier + default: Number | None = None class ConstantOperand(Requirements): @@ -102,7 +130,15 @@ class ConstantOperand(Requirements): value: Number -Operand = Annotated[FieldOperand | ConstantOperand, Field(discriminator="kind")] +class SumOperand(Requirements): + kind: Literal[OperandKind.SUM] + operands: tuple["Operand", ...] = Field(min_length=1) + + +Operand = Annotated[ + FieldOperand | ConstantOperand | SumOperand, Field(discriminator="kind") +] +SumOperand.model_rebuild() class CountWhen(Requirements): @@ -117,46 +153,48 @@ class CountOverride(Requirements): class AccuracyCoverageRequirements(Requirements): mandatory_bands: Identifier - margin_counts: StrictBool | None = None + margin_counts: StrictBool = False minimum_results_per_band: NonnegativeInt single_turn_requires_offline_accuracy: StrictBool | None = None class AccuracyGateRequirements(Requirements): - aggregation: Identifier | None = None + aggregation: Literal[AccuracyMode.SAMPLE_WEIGHTED_MEAN] | None = None bands: Identifier | None = None bounds: Identifier | None = None - count_scope: Identifier | None = None + count_scope: Literal[AccuracyMode.PER_REQUIRED_DATASET_OR_SUITE] | None = None dataset: Identifier | None = None dataset_catalog: Identifier | None = None - fraction_conversion: Identifier | None = None + fraction_conversion: FractionConversion | None = None inclusive: StrictBool | None = None input_range: tuple[Number, Number] | None = None - metric_matching: Identifier | None = None + metric_matching: Literal[AccuracyMode.CASE_INSENSITIVE] | None = None minimum: NonnegativeNumber | Identifier | None = None minimum_repeats: PositiveInt | None = None missing_repeats: PositiveInt | None = None models: Identifier - on_missing: Identifier | None = None - on_missing_bands: Identifier | None = None - on_missing_metric: Identifier | None = None - on_missing_required_dataset: Identifier | None = None - on_missing_weights: Identifier | None = None - on_multiple_results_per_band: Identifier | None = None - on_no_results: Identifier | None = None - on_no_sample_counts: Identifier | None = None - on_unknown_model: Identifier | None = None - on_unknown_or_unpublished: Identifier | None = None - on_unrelated_dataset: Identifier | None = None + on_missing: ( + Literal[PolicyAction.ERROR, PolicyAction.SKIP, PolicyAction.WARNING] | None + ) = None + on_missing_bands: Literal[PolicyAction.GATE_AVAILABLE_MEAN_AND_WARN] | None = None + on_missing_metric: Literal[PolicyAction.SKIP, PolicyAction.ERROR] | None = None + on_missing_required_dataset: Literal[PolicyAction.ERROR] | None = None + on_missing_weights: Literal[AccuracyMode.ARITHMETIC_MEAN] | None = None + on_multiple_results_per_band: Literal[PolicyAction.AVERAGE_AND_WARN] | None = None + on_no_results: Literal[PolicyAction.WARNING] | None = None + on_no_sample_counts: Literal[PolicyAction.SKIP] | None = None + on_unknown_model: Literal[PolicyAction.WARNING] | None = None + on_unknown_or_unpublished: Literal[PolicyAction.WARNING] | None = None + on_unrelated_dataset: Literal[PolicyAction.EXCLUDE] | None = None operation: AccuracyOperation output_scale: PositiveNumber | None = None required_band_count: PositiveInt | None = None required_datasets: Identifier | None = None - score: Identifier | None = None + score: Literal[AccuracyMode.SCORE_OR_FIRST_METRIC] | None = None source: Identifier | None = None threshold: Identifier | None = None - units: Identifier | None = None - unnamed_scalar: Identifier | None = None + units: Literal[AccuracyMode.SAMPLES_TIMES_REPEATS] | None = None + unnamed_scalar: Literal[AccuracyMode.SOLE_METRIC_ONLY_WITH_WARNING] | None = None windowed_fallback: StrictBool | None = None @model_validator(mode="after") @@ -184,20 +222,22 @@ def check_form(self) -> Self: class AccuracyPresenceRequirements(Requirements): - accept: list[Identifier] + accept: tuple[AccuracyPresenceSource, ...] minimum_models: NonnegativeInt class ArtifactBindingRequirements(Requirements): catalog: Identifier - format: Identifier | None = None - matching: BindingMatch - on_empty_approval_list: Identifier | None = None - on_empty_model_catalog: Identifier | None = None - on_missing: Identifier | None = None + format: Literal[IdentifierFormat.GIT_COMMIT_SHA1] | None = None + matching: Literal[BindingMatch.CHECKPOINT, BindingMatch.CLIENT_ALLOWLIST] + on_empty_approval_list: Literal[PolicyAction.BLOCKED] | None = None + on_empty_model_catalog: Literal[PolicyAction.BLOCKED] | None = None + on_missing: ( + Literal[PolicyAction.ERROR, PolicyAction.SKIP, PolicyAction.WARNING] | None + ) = None repository: Identifier | None = None source: Identifier | None = None - sources: list[Identifier] | None = None + sources: tuple[Identifier, ...] | None = None @model_validator(mode="after") def check_form(self) -> Self: @@ -214,28 +254,35 @@ def check_form(self) -> Self: class ArtifactSchemaRequirements(Requirements): - accept: list[Identifier] | None = None + accept: tuple[AccuracyArtifactSource, ...] | None = None artifact: Identifier | None = None - precedence: Identifier | None = None + precedence: Literal[ArtifactPrecedence.STANDALONE_FIRST] | None = None reject_empty: StrictBool | None = None - artifact_schema: Identifier = Field(alias="schema") + artifact_schema: Literal[ + ArtifactSchema.ACCURACY_RESULT, + ArtifactSchema.POINT_CONFIG, + ArtifactSchema.RESULT_SUMMARY, + ArtifactSchema.SYSTEM_DESCRIPTION, + ] = Field(alias="schema") when_present: StrictBool | None = None class CatalogIntegrityRequirements(Requirements): catalog: Identifier - on_unavailable: Identifier | None = None + on_unavailable: ( + Literal[PolicyAction.BLOCKED, PolicyAction.BLOCK_DEPENDENT_CHECKS] | None + ) = None class CohortIdentifierRequirements(Requirements): - format: Identifier + format: Literal[IdentifierFormat.YEAR_MONTH_C0_OR_C1] target: Identifier class CollectionSizeRequirements(Requirements): maximum: NonnegativeInt | None = None minimum: NonnegativeInt | None = None - overrides: list[CountOverride] | None = None + overrides: tuple[CountOverride, ...] = () source: Identifier @model_validator(mode="after") @@ -259,11 +306,15 @@ class ComparisonRequirements(Requirements): class ConsistencyRequirements(Requirements): - comparison: Identifier | None = None - exclude_fields: list[Identifier] | None = None + comparison: ( + Literal[ConsistencyMode.EXACT, ConsistencyMode.WHOLE_DOCUMENT] | None + ) = None + exclude_fields: tuple[Identifier, ...] | None = None left: Identifier | None = None - on_disagreement_curve_type: Identifier | None = None - on_missing: Identifier | None = None + on_disagreement_curve_type: Literal[CurveTypePolicy.SINGLE_TURN] | None = None + on_missing: ( + Literal[PolicyAction.ERROR, PolicyAction.SKIP, PolicyAction.WARNING] | None + ) = None require_nonempty: StrictBool | None = None required_artifact: Identifier | None = None right: Identifier | None = None @@ -281,7 +332,7 @@ def check_form(self) -> Self: class CoverageRequirements(Requirements): band: Identifier | Bounds - margin_counts: StrictBool | None = None + margin_counts: StrictBool = False minimum_count: NonnegativeInt target: Identifier @@ -290,16 +341,18 @@ class CountRequirements(Requirements): catalog: Identifier dataset: Identifier | None = None dataset_source: Identifier | None = None - on_missing: Identifier | None = None - on_unknown_or_null_threshold: Identifier | None = None - on_unsupported_sample_unit: Identifier | None = None + on_missing: ( + Literal[PolicyAction.ERROR, PolicyAction.SKIP, PolicyAction.WARNING] | None + ) = None + on_unknown_or_null_threshold: Literal[PolicyAction.SKIP] | None = None + on_unsupported_sample_unit: Literal[PolicyAction.SKIP] | None = None operator: CountOperator require_all: StrictBool | None = None sample_unit: SampleUnit | None = None source: Identifier | None = None - sources: list[Identifier] | None = None - supported_sample_units: list[SampleUnit] | None = None - threshold: Identifier | None = None + sources: tuple[Identifier, ...] | None = None + supported_sample_units: tuple[SampleUnit, ...] | None = None + threshold: Identifier = "sample_count" @model_validator(mode="after") def check_form(self) -> Self: @@ -312,15 +365,17 @@ class DerivedMetricRequirements(Requirements): constants: Identifier | None = None denominator: Identifier | None = None numerator: PositiveNumber | None = None - on_invalid_tpot: Identifier | None = None - on_missing_or_invalid_file: Identifier | None = None - on_missing_or_nonpositive_power: Identifier | None = None - on_missing_stored: Identifier | None = None - on_no_inputs_and_no_stored: Identifier | None = None - on_nonpositive_peak: Identifier | None = None - on_stored_without_derivable_inputs: Identifier | None = None + on_invalid_tpot: Literal[PolicyAction.SKIP] | None = None + on_missing_or_invalid_file: Literal[PolicyAction.SKIP] | None = None + on_missing_or_nonpositive_power: Literal[PolicyAction.SKIP] | None = None + on_missing_stored: Literal[PolicyAction.REPORT_DERIVED] | None = None + on_no_inputs_and_no_stored: Literal[PolicyAction.SKIP] | None = None + on_nonpositive_peak: Literal[PolicyAction.SKIP] | None = None + on_stored_without_derivable_inputs: Literal[PolicyAction.ERROR] | None = None operation: DerivedOperation - peak_source: Identifier | None = None + peak_source: ( + Literal[RegionSource.POINTS_WITH_READABLE_THROUGHPUT_AND_UTILIZATION] | None + ) = None source: Identifier | None = None stored: Identifier tolerance: Tolerance | None = None @@ -340,27 +395,27 @@ class DisclosureRequirements(Requirements): fields: Identifier -class DrafterBindingRequirements(Requirements): +class SpecDecodeHeadRequirements(Requirements): catalog: Identifier - matching: BindingMatch | None = None + matching: Literal[BindingMatch.SPEC_DECODE_HEAD] | None = None minimum_cohorts: PositiveInt | None = None - on_empty_catalog: Identifier | None = None - on_missing_approval_cohort: Identifier | None = None - on_missing_repository_or_revision: Identifier | None = None - on_unmatched_drafter: Identifier | None = None - on_unparseable_cohort: Identifier | None = None - operation: DrafterOperation + on_empty_catalog: Literal[PolicyAction.ERROR] | None = None + on_missing_approval_cohort: Literal[PolicyAction.WARNING] | None = None + on_missing_identity: Literal[PolicyAction.ERROR] | None = None + on_unmatched_spec_decode_head: Literal[PolicyAction.SKIP] | None = None + on_unparseable_cohort: Literal[PolicyAction.WARNING] | None = None + operation: SpecDecodeHeadOperation @model_validator(mode="after") def check_form(self) -> Self: - if self.operation is DrafterOperation.APPROVAL_AGE: + if self.operation is SpecDecodeHeadOperation.APPROVAL_AGE: self.require_fields("minimum_cohorts") return self class DurationRequirements(Requirements): - basis_precedence: list[DurationBasis] - on_unclassifiable_concurrency: Identifier | None = None + basis_precedence: tuple[DurationBasis, ...] + on_unclassifiable_concurrency: Literal[PolicyAction.SKIP] | None = None thresholds: Identifier window_status_required: StrictBool | None = None @@ -368,7 +423,9 @@ class DurationRequirements(Requirements): class FieldConstraintsRequirements(Requirements): constraints: Identifier | None = None constraints_by_model: Identifier | None = None - on_missing: Identifier | None = None + on_missing: ( + Literal[PolicyAction.ERROR, PolicyAction.SKIP, PolicyAction.WARNING] | None + ) = None target: Identifier | None = None @model_validator(mode="after") @@ -380,14 +437,14 @@ def check_form(self) -> Self: class IssuanceRequirements(Requirements): - allowed: list[LoadPatternType] + allowed: tuple[LoadPatternType, ...] require_positive_concurrency: StrictBool | None = None class MembershipRequirements(Requirements): catalog: Identifier - deduplicate: StrictBool | Identifier | None = None - matching: Identifier | None = None + deduplicate: StrictBool | Deduplication | None = None + matching: Literal[ConsistencyMode.EXACT] | None = None required: StrictBool | None = None skip_missing: StrictBool | None = None target: Identifier @@ -405,11 +462,11 @@ class OfflineRequirements(Requirements): agentic_count: NonnegativeInt | None = None concurrency_floor: NonnegativeNumber | Identifier | None = None elected_must_equal: PositiveInt | Identifier | None = None - on_missing_summary: Identifier | None = None + on_missing_summary: Literal[PolicyAction.SKIP] | None = None operation: OfflineOperation single_turn_count: NonnegativeInt | None = None throughput_minimum_multiplier: NonnegativeNumber | None = None - throughput_reference: Identifier | None = None + throughput_reference: Literal[RegionSource.C_MAX_POINT] | None = None @model_validator(mode="after") def check_form(self) -> Self: @@ -423,40 +480,44 @@ def check_form(self) -> Self: class PathResolutionRequirements(Requirements): - allow_absolute: StrictBool | None = None - allow_escape_via_symlink: StrictBool | None = None - allow_parent_traversal: StrictBool | None = None + allow_absolute: StrictBool = False + allow_escape_via_symlink: StrictBool = False + allow_parent_traversal: StrictBool = False base: Identifier - on_missing: Identifier | None = None + on_missing: ( + Literal[PolicyAction.ERROR, PolicyAction.SKIP, PolicyAction.WARNING] | None + ) = None require_directory: StrictBool | None = None - targets: list[Identifier] + targets: tuple[Identifier, ...] class PowerRequirements(Requirements): artifact: Identifier | None = None constants: Identifier | None = None cooling_must_match_system_description: StrictBool | None = None - declared_total_scaling: Identifier | None = None - on_disaggregated: Identifier | None = None - on_missing_parallelism: Identifier | None = None - on_spare_nodes: Identifier | None = None - on_undeclared_nodes: Identifier | None = None - on_unknown_accelerator_count: Identifier | None = None + declared_total_scaling: Literal[PowerMode.MAXIMUM_ENGAGED_FRACTION] | None = None + on_disaggregated: Literal[PolicyAction.WARNING] | None = None + on_missing_parallelism: Literal[PolicyAction.WARNING] | None = None + on_spare_nodes: Literal[PolicyAction.WARNING] | None = None + on_undeclared_nodes: Literal[PowerMode.FULLY_ENGAGED] | None = None + on_unknown_accelerator_count: Literal[PolicyAction.WARNING] | None = None operation: PowerOperation parallelism: Identifier | None = None reject_duplicate_ensemble_ids: StrictBool | None = None - replica_factors: list[Identifier] | None = None + replica_factors: tuple[Identifier, ...] | None = None require_accelerator_capacity: StrictBool | None = None require_derivable_total: StrictBool | None = None require_known_ensemble_ids: StrictBool | None = None require_maximum_data_parallel: StrictBool | None = None require_nodes_within_provisioned: StrictBool | None = None round_kw_decimals: NonnegativeInt | None = None - artifact_schema: Identifier | None = Field(default=None, alias="schema") + artifact_schema: Literal[ArtifactSchema.APPENDIX_E_SYSTEM_POWER] | None = Field( + default=None, alias="schema" + ) shortfall: Identifier | None = None source: Identifier | None = None - switch_scaling: Identifier | None = None - tag: Identifier | None = None + switch_scaling: Literal[PowerMode.ENGAGED_NODE_FRACTION] | None = None + tag: Literal[PowerTag.ESTIMATED_POWER] | None = None validate_computed_values: StrictBool | None = None validate_node_sets: StrictBool | None = None validate_public_sources: StrictBool | None = None @@ -475,11 +536,19 @@ def check_form(self) -> Self: class PresenceRequirements(Requirements): case_sensitive: StrictBool | None = None - minimum_count: NonnegativeInt | None = None - object: Identifier | None = None + minimum_count: NonnegativeInt = 1 + object: ( + Literal[ + PresenceObject.DECLARATION, + PresenceObject.DIRECTORY, + PresenceObject.FILE, + PresenceObject.PATH, + ] + | None + ) = None pattern: Identifier | None = None target: Identifier | None = None - targets: list[Identifier] | None = None + targets: tuple[Identifier, ...] | None = None @model_validator(mode="after") def check_form(self) -> Self: @@ -488,9 +557,9 @@ def check_form(self) -> Self: class RegionBasisRequirements(Requirements): - minimum_concurrency: Identifier | None = None - on_no_parsed_points: Identifier | None = None - on_partial_parse: Identifier | None = None + minimum_concurrency: Literal[RegionSource.SMALLEST_SUBMITTED] | None = None + on_no_parsed_points: Literal[PolicyAction.ERROR] | None = None + on_partial_parse: Literal[PolicyAction.WARNING] | None = None source: Identifier upper_clamp: PositiveInt @@ -502,31 +571,41 @@ class RegionBoundariesRequirements(Requirements): class RegionPlacementRequirements(Requirements): - ignore_declared: list[Identifier] | None = None + ignore_declared: tuple[Identifier, ...] = () include_margin: StrictBool | None = None - on_missing_or_unclassifiable: Identifier | None = None + on_missing_or_unclassifiable: Literal[PolicyAction.SKIP] | None = None operation: RegionPlacementOperation class ReportRequirements(Requirements): - interpretation: Identifier | None = None - operation: Identifier | None = None + interpretation: Literal[ReportInterpretation.CLIENT_IPC_FORWARDING_ONLY] | None = ( + None + ) + operation: Literal[ReportOperation.BLOCKED_CONFIG_DEPENDENTS] | None = None target: Identifier class SeedBindingRequirements(Requirements): adoption_window_cohorts: PositiveInt | None = None - aliases: dict[Identifier, Identifier] | None = None + aliases: Mapping[Identifier, Identifier] | None = None catalog: Identifier | None = None - fields: list[Identifier] | None = None - on_legacy_registry_without_cohorts: Identifier | None = None - on_unavailable: Identifier | None = None + fields: tuple[Identifier, ...] | None = None + on_legacy_registry_without_cohorts: Literal[PolicyAction.BLOCKED] | None = None + on_unavailable: ( + Literal[PolicyAction.BLOCKED, PolicyAction.BLOCK_DEPENDENT_CHECKS] | None + ) = None operation: SeedOperation require_all: StrictBool | None = None warn_legacy_only_when_model_seed_missing: StrictBool | None = None + @field_serializer("aliases") + def serialize_aliases(self, value): + return dict(value) if value is not None else None + @model_validator(mode="after") def check_form(self) -> Self: + if self.aliases is not None: + object.__setattr__(self, "aliases", MappingProxyType(dict(self.aliases))) if self.operation is SeedOperation.LEGACY_NAMES: self.require_fields("aliases") else: @@ -541,19 +620,21 @@ def check_form(self) -> Self: class SteadyStateRequirements(Requirements): check_reported_super_pass_count: StrictBool | None = None constants: Identifier - fallback: Identifier | None = None + fallback: Literal[ReportingBasis.WHOLE_RUN_TOTAL] | None = None official_requires_plateau: StrictBool | None = None - on_drift: Identifier | None = None - on_missing_or_nonofficial: Identifier | None = None - on_missing_window_extent: Identifier | None = None + on_drift: Literal[PolicyAction.WARN_RANGE_OR_SLOPE_REQUIRED] | None = None + on_missing_or_nonofficial: Literal[PolicyAction.WARNING] | None = None + on_missing_window_extent: Literal[PolicyAction.SKIP_COUNT_COMPARISON] | None = None operation: SteadyOperation class WarmupRequirements(Requirements): - disabled_warmup: Identifier | None = None - on_missing: Identifier | None = None + disabled_warmup: Literal[PolicyAction.EXEMPT] | None = None + on_missing: ( + Literal[PolicyAction.ERROR, PolicyAction.SKIP, PolicyAction.WARNING] | None + ) = None operation: WarmupOperation - require_declared_retained: StrictBool | None = None + require_declared_retained: StrictBool = True source: Identifier | None = None verify_archive_contents: StrictBool | None = None @@ -579,7 +660,7 @@ def check_form(self) -> Self: CheckKind.COUNT: CountRequirements, CheckKind.DERIVED_METRIC: DerivedMetricRequirements, CheckKind.DISCLOSURE: DisclosureRequirements, - CheckKind.DRAFTER_BINDING: DrafterBindingRequirements, + CheckKind.SPEC_DECODE_HEAD: SpecDecodeHeadRequirements, CheckKind.DURATION: DurationRequirements, CheckKind.FIELD_CONSTRAINTS: FieldConstraintsRequirements, CheckKind.ISSUANCE: IssuanceRequirements, diff --git a/src/inference_endpoint/validation/schemas/v1.py b/src/inference_endpoint/validation/schemas/v1.py index 064663dd3..dc28cb137 100644 --- a/src/inference_endpoint/validation/schemas/v1.py +++ b/src/inference_endpoint/validation/schemas/v1.py @@ -113,7 +113,7 @@ def __call__(self, documents, digests) -> Policy: if key not in Metadata.model_fields } try: - validate_requirements(metadata.kind, parameters) + requirements = validate_requirements(metadata.kind, parameters) except ValueError as error: raise ValueError(f"Rule {id_}: {error}") from error validate_catalog_references( @@ -137,7 +137,7 @@ def __call__(self, documents, digests) -> Policy: metadata.applies_to, metadata.unless, frozenset(metadata.requires), - freeze(parameters), + requirements, references, file, needs_model(parameters), @@ -151,7 +151,7 @@ def __call__(self, documents, digests) -> Policy: rule = rules[override.rule] try: validate_requirements( - rule.kind, {**rule.requirements, **override.requirements} + rule.kind, {**rule.requirements.wire(), **override.requirements} ) except ValueError as error: raise ValueError( diff --git a/src/inference_endpoint/validation/schemas/values_v1.py b/src/inference_endpoint/validation/schemas/values_v1.py index dc4e4b8d2..86a729819 100644 --- a/src/inference_endpoint/validation/schemas/values_v1.py +++ b/src/inference_endpoint/validation/schemas/values_v1.py @@ -11,6 +11,7 @@ from ..types import ( AccuracyKind, + ChecksumValue, CohortId, CommitId, Cooling, @@ -37,13 +38,23 @@ class Checkpoint(FrozenModel): revision: CommitId -class DecodeHead(Checkpoint): +class SpecDecodeHead(Checkpoint): model: Identifier method: DecodeMethod bundled: bool = Field(default=False, strict=True) approved_cohort: CohortId | None = None +class ConfigurationSpecDecodeHead(FrozenModel): + """An approved draft pass identified by target weights and configuration.""" + + model: Identifier + method: DecodeMethod + target_checksum: ChecksumValue + configuration: dict[Identifier, JsonValue] = Field(min_length=1) + approved_cohort: CohortId | None = None + + class Accuracy(FrozenModel): kind: AccuracyKind datasets: tuple[Identifier, ...] = Field(min_length=1) @@ -99,7 +110,7 @@ class Catalogs(FrozenModel): mandatory_accuracy_bands: tuple[Identifier, ...] datasets: dict[Identifier, Dataset] models: dict[Identifier, Model] - approved_sped_decode_heads: tuple[DecodeHead, ...] + approved_spec_decode_heads: tuple[SpecDecodeHead | ConfigurationSpecDecodeHead, ...] approved_client_revisions: tuple[CommitId, ...] approved_checkpoints: dict[Identifier, tuple[Checkpoint, ...]] submission_flags: dict[str, JsonValue] @@ -131,7 +142,7 @@ def check_references(self): raise ValueError(f"Unknown accuracy dataset for {name}") if not self.approved_checkpoints.keys() <= self.models.keys(): raise ValueError("Checkpoint catalog references unknown models") - for head in self.approved_sped_decode_heads: + for head in self.approved_spec_decode_heads: if head.model not in self.models: raise ValueError("Decode head references unknown model") return self diff --git a/src/inference_endpoint/validation/types.py b/src/inference_endpoint/validation/types.py index e700d423e..50b51fcd2 100644 --- a/src/inference_endpoint/validation/types.py +++ b/src/inference_endpoint/validation/types.py @@ -19,6 +19,9 @@ def _canonical_identifier(value: str) -> str: ] CohortId = Annotated[str, StringConstraints(pattern=r"^\d{4}-(0[1-9]|1[0-2])-C[01]$")] CommitId = Annotated[str, StringConstraints(pattern=r"^[0-9a-f]{40}$")] +ChecksumValue = Annotated[ + str, StringConstraints(pattern=r"^(git-sha1:[0-9a-f]{40}|sha256:[0-9a-f]{64})$") +] class FrozenModel(BaseModel): @@ -86,7 +89,7 @@ class EvidenceKey(StrEnum): WARMUP = "warmup" STEADY_STATE = "steady_state" SEED_CATALOG = "seed_catalog" - APPROVED_SPED_DECODE_HEADS = "approved_sped_decode_heads" + APPROVED_SPEC_DECODE_HEADS = "approved_spec_decode_heads" POINT_CONFIG = "point_config" POWER_COMPUTATION = "power_computation" POINT_POWER = "point_power" @@ -118,6 +121,7 @@ class SampleUnit(StrEnum): class DecodeMethod(StrEnum): DSPARK = "dspark" MTP = "mtp" + SELF_SPECULATIVE = "self_speculative" class Cooling(StrEnum): diff --git a/src/inference_endpoint/validation/vocabulary.py b/src/inference_endpoint/validation/vocabulary.py index 8bde0ccb9..cb26b8d47 100644 --- a/src/inference_endpoint/validation/vocabulary.py +++ b/src/inference_endpoint/validation/vocabulary.py @@ -20,7 +20,7 @@ class CheckKind(StrEnum): COUNT = "count" DERIVED_METRIC = "derived_metric" DISCLOSURE = "disclosure" - DRAFTER_BINDING = "drafter_binding" + SPEC_DECODE_HEAD = "spec_decode_head" DURATION = "duration" FIELD_CONSTRAINTS = "field_constraints" ISSUANCE = "issuance" diff --git a/tests/fixtures/validation/policy-fingerprints.json b/tests/fixtures/validation/policy-fingerprints.json index 66dbbaaa7..9f733630f 100644 --- a/tests/fixtures/validation/policy-fingerprints.json +++ b/tests/fixtures/validation/policy-fingerprints.json @@ -1,41 +1,8 @@ { - "version": "2026-10-C1", - "revision": 1, - "catalog_digest": "b6d1e4363c4cfd80a255951f3f08fd8a21913918fb586ced16eff6b51e7beee6", + "catalog_digest": "3d3bbc0f32b62e5cd7567da203a20f2edf382cd4ebf728ac090f935f3f8ec51b", "enrollment_digest": "4d4f3a4087945435ffacee743906a9597be630d8117aca1a26bf0a4bd3a11cab", + "revision": 1, "rules": { - "submission/path-exists": "10d1f797fff02890ddd7e3473732b96c44520d23952f9b7493fc576f6e5bf6a3", - "submission/required-dir": "17674a08b2897de6897d22270998e999d0ff8ac118a35d263555604c727a3f9d", - "submission/system-results-dir": "72456038d0d00ed12a7e3d58eda518b86b69aea7b019fad32b13a8310359704c", - "submission/src-dir": "a5b4a62887d8636f2f758a717100b68bfe31f3107648536236e990569f971cfa", - "submission/src-readme": "5754b3df0d5389985d5a476d630274d9fa1143e6cfa04be8616e119a55b5ac3b", - "submission/accuracy-present": "2f0dc1d59739717847bcb1519c78d6f2bbd871e9a1e666e6daa99b31f0a0824c", - "submission/system-name-consistency": "7ef9d2b5a293776dd30c1fea663e1b401678c75f826d89743cf0706a628784e6", - "system/benchmark-model-dir": "517f85dedefe201e639a74d83c581cd9efa481731848761d9ade76df7326bb3e", - "system/power-descriptor": "7b3b36310067d0772fb58c927e219d68c73881c25f62637fcb98498b6e9639a7", - "system/power-estimated": "d561adae981297c84fe746712f3e4008857e2e9c947c9d69f41e6595c3409fd3", - "system/system-description-across-curves": "6ff9a22fb2d7bc71fcdb91c7a13c8bee6bec0f8cfaefc5c5f1ad80c901c23f43", - "curve/point-dirs": "7f5c829ce0379813c1a490dcda57449675767ba7b0bd6849ab91d8ad3c2233ef", - "curve/measurement-points-present": "dafd79462c87ff519281ad09ae256ddafee22f2c681c46c60589dabfa514c464", - "curve/system-description-consistency": "77e9c8d89f667c8f830edd8ddf77191d177cab78a4ad2291e14e185b72c29782", - "curve/max-concurrency-declared": "54b6c7e9800b8bfe5099da91ffaecb25e462fccecf5098a89a6cfef10aeeb0eb", - "curve/region-basis": "2b2431e4f01f66b9a77fc7ecab0e8e90e8e76a7f83663eeb235dbdb08116ba54", - "curve/region-computation": "24c56d29544e482c4f9931f399ade3d232fc2744c77c16174d258dd74a091a21", - "curve/model-name-valid": "b3a33471bb36c3bb0d57eb3b8d015f95e2176aa530c85ce547a72dd0b560f788", - "curve/model-name-consistency": "8c6cac2996bacf763af57c2e654846dc52cea95345d495900fc7f35605a2147f", - "curve/config-consistency-model": "709507fdd16f1788b19ae7f6d7a3893417d8bc73f4acb24dcdafc95a843d1296", - "curve/config-consistency-dataset": "97539f45fbb7fdf9916bb16ec24607e5a194092186e2934601fc4d922a9ea5e2", - "curve/benchmark-type-consistency": "b2e5d794ae9e39c619b436cf01318b78651080a4b9f9bd993db536c3878f6c10", - "curve/point-count": "2314996964b653a78641df73cfac4c341f6c1260107e435eff0d982f4ea9ff15", - "curve/point-cap": "ef4565bdbce9fe5d548d4c758bd06acfdaf8614e032018f0cb22c2bd7b122cea", - "curve/ultra-low-concurrency-coverage": "a83bba002e1be6cdb4597b2e27597e97ed79abd1157ca9c1d7d285de4516486d", - "curve/low-concurrency-coverage": "c23a35ad020b892b5e769675433a4f3bd815a454d7a54800970675e81262e597", - "curve/med-concurrency-coverage": "1fa4671c42fe8a7b92f036e220419b7e85874cd132b76414146907e03c0e8166", - "curve/high-concurrency-coverage": "ad1180bb8571047a8c51aa297cfee8ff9f0ced96b1a732b3fabec97b1bfc4e14", - "curve/offline-point-present": "a8d496b614e4010ddfffe2799ff3585af83db6a28f21c92eadd47443360d66bf", - "curve/offline-ordering": "6580ae22479293be5d268977c0aec7eb6b0b09b93b7940cda1092af7d3cc246c", - "curve/tps-utilization": "000e3e3938029ed129495ad661900103d8b7daf49aaf9a463cef0333025b8e1b", - "curve/seed-set-consistency": "8d448c73d6012d37571a4786e3f2661669e7a0e4b787ec213d90bb8805fef215", "curve/accuracy-coverage": "eb5722a35f0682b5da082689e02b1828fba8124259b10ef83e71a56ae45559e1", "curve/accuracy-gate": "eaea400ff65e395339cf207162039a23f49c0a7abc8ee44c9b9714fc0cd8a98f", "curve/accuracy-sample-count": "e36c67cb755ee8f69eef331574785d4e23c1faaa324ea6f01fa3226d6c5d8f22", @@ -43,55 +10,88 @@ "curve/agentic-accuracy-inline": "cc92cf9b4be30ce05c95d46da6bc1cb07d6098272224b554d05daf1975b8c09f", "curve/agentic-accuracy-swebench": "af5ba468e3c4af612c1a0fb21006b82fdb68b2e648045df2d56c41734da0d79f", "curve/agentic-osl-range": "e20feb64472b8faddcf31e08f3c18836c419ff239d942026bf275b0147597074", - "point/submission-flags": "6fd56145219a078dd947d79bd1a5587e9c45da258085ea0e0145d3d75e7330a9", - "point/swebench-template": "664ef45ae9124f181b8b1e947e25f29b6567ebb09c4ea192052c50678435dc15", - "point/trajectories-to-issue": "323a70bf8bca223b7f926afb53bfeacb519f59cc51949484b3643642bf2e79ab", - "point/swebench-instance-count": "84477a7f4dc7e0da91fabdf4e7ec73cbd886194eeadb3895136e4594980304fc", - "point/endpoints-client-sha": "23b06e7e9a8dd4b39cf1cbd6809ec3f5bc2c80a49f460011b62f6dcb70ed8536", - "point/approved-checkpoint": "fd35d43f5c4c9f4bb268a80c6d365226dfc53a25902fa98bf378add73e4766c3", - "point/point-power": "6e75d380e17c6198e37ade0ac1ebd4598fc57c367a7994ff9543a75cb2106961", - "point/point-config-valid": "4bb6c62e68b0748f1a1b47110f0083029d0024958d17f4b8db38af21aed9dfaf", - "point/point-dirname-concurrency": "2296b404dacb1a6f15106be0dc2a6acea4d519d54e44c486e35c6ac78ec066fc", - "point/system-description-present": "856c0e732cba243a1bb153ab39176653a0d3a33fd33958095e564cd0e529f785", - "point/system-description-valid": "b53aee4801f00277cece609d210b3ce4496a257fe0d1d6a3ea2bda7b24b80c1d", - "point/result-summary-present": "9eaf56fa9c6a152f94563d2d8555a5435b887c7db9d3d9c5769c2f87c5b971d8", - "point/result-file-valid": "e5cebe2ee4158e9630ada483b0ac6596c3ee58608313a152ba8a460337f9f5f8", + "curve/benchmark-type-consistency": "b2e5d794ae9e39c619b436cf01318b78651080a4b9f9bd993db536c3878f6c10", + "curve/config-consistency-dataset": "97539f45fbb7fdf9916bb16ec24607e5a194092186e2934601fc4d922a9ea5e2", + "curve/config-consistency-model": "709507fdd16f1788b19ae7f6d7a3893417d8bc73f4acb24dcdafc95a843d1296", + "curve/high-concurrency-coverage": "ad1180bb8571047a8c51aa297cfee8ff9f0ced96b1a732b3fabec97b1bfc4e14", + "curve/low-concurrency-coverage": "c23a35ad020b892b5e769675433a4f3bd815a454d7a54800970675e81262e597", + "curve/max-concurrency-declared": "54b6c7e9800b8bfe5099da91ffaecb25e462fccecf5098a89a6cfef10aeeb0eb", + "curve/measurement-points-present": "dafd79462c87ff519281ad09ae256ddafee22f2c681c46c60589dabfa514c464", + "curve/med-concurrency-coverage": "1fa4671c42fe8a7b92f036e220419b7e85874cd132b76414146907e03c0e8166", + "curve/model-name-consistency": "8c6cac2996bacf763af57c2e654846dc52cea95345d495900fc7f35605a2147f", + "curve/model-name-valid": "b3a33471bb36c3bb0d57eb3b8d015f95e2176aa530c85ce547a72dd0b560f788", + "curve/offline-ordering": "6580ae22479293be5d268977c0aec7eb6b0b09b93b7940cda1092af7d3cc246c", + "curve/offline-point-present": "a8d496b614e4010ddfffe2799ff3585af83db6a28f21c92eadd47443360d66bf", + "curve/point-cap": "ef4565bdbce9fe5d548d4c758bd06acfdaf8614e032018f0cb22c2bd7b122cea", + "curve/point-count": "2314996964b653a78641df73cfac4c341f6c1260107e435eff0d982f4ea9ff15", + "curve/point-dirs": "7f5c829ce0379813c1a490dcda57449675767ba7b0bd6849ab91d8ad3c2233ef", + "curve/region-basis": "2b2431e4f01f66b9a77fc7ecab0e8e90e8e76a7f83663eeb235dbdb08116ba54", + "curve/region-computation": "24c56d29544e482c4f9931f399ade3d232fc2744c77c16174d258dd74a091a21", + "curve/seed-set-consistency": "8d448c73d6012d37571a4786e3f2661669e7a0e4b787ec213d90bb8805fef215", + "curve/system-description-consistency": "77e9c8d89f667c8f830edd8ddf77191d177cab78a4ad2291e14e185b72c29782", + "curve/tps-utilization": "000e3e3938029ed129495ad661900103d8b7daf49aaf9a463cef0333025b8e1b", + "curve/ultra-low-concurrency-coverage": "a83bba002e1be6cdb4597b2e27597e97ed79abd1157ca9c1d7d285de4516486d", + "diagnostics/seed-set-registry": "7670c90fb34c6459fc0a1808c999337db09d7e7d9a95283521d9235cddd4d647", + "diagnostics/spec-decode-head-catalog": "775556b490476a6fb5b4738f83caa839e7b9a591340423fe0655171171b70d80", "point/accuracy-valid": "3ac1f69a8b6103b319326669d255fae3c282888a188717dc2c9e5b73bdbdcbc6", - "point/point-disclosure-complete": "218ae026f7fdec937b0161af67519e60698a087c52eac59e579ae77203b254be", - "point/target-cohort": "6855eb240be6164648dccd441741230bca1783680c0b53a67cdf6b54e5466eba", - "point/shared-path-resolution": "4f6901ad243563cad240c70a7f0afed406639e5ee6d2f5ef4ea15b3ebc88ff14", - "point/offline-declared": "71281f294529c5665bde54020b1f8a5e204b9ed5735f9a135458eb2cae3c6a5a", - "point/region-declared": "e4340e52fed15f4a3dee74b73086f6688344057adddc45d737fb0bb88475719c", + "point/agentic-metric-consistency": "2c08e062819cfb8f397938cc0c96030dcb34fb379e0681a0e79d211e6c768731", + "point/approved-checkpoint": "fd35d43f5c4c9f4bb268a80c6d365226dfc53a25902fa98bf378add73e4766c3", + "point/approved-spec-decode-head": "e8a2254ac5e78d036f5ff1116171e515bc49b77a3e851b40ac6a5acc0ffaa40f", "point/concurrency-in-range": "92b745ed6eba6d607bdaecae8f911838eda0940093f9422765f9fd3a374b44fe", - "point/region-placement": "283c684b2b3276f211af1a45f78f07702e804560a05a3e9993a05c58ad1c8283", + "point/endpoints-client-sha": "23b06e7e9a8dd4b39cf1cbd6809ec3f5bc2c80a49f460011b62f6dcb70ed8536", "point/load-pattern": "615a4d3f2faf27f07760c032cde07715ac17af76c9b5c1728635b45b8eeacace", - "point/streaming-config": "6b7d56890aed2d66ff24c2c1ee8a9b1b8cd4d465f94e8dc800b76ea9632f2fd9", - "point/warmup-present": "ade8ebb85859a4824b300ce75598921791c5b7f9cd9a9569d4729f27404aced1", - "point/warmup-logs-retained": "d63e4b1891824a0834adca73b95a192bbf3685536bbb4161f21d1a4cc72e7de4", - "point/warmup-salt": "091a57e9035d9a7f98e64cec9429bb5ed5b5a5a1b1f3b4f8510e191b7bc111f8", - "point/steady-state-valid": "39699685e3a49ff6db27ee6c02405878ef2362262f03f37088ac2a4a42ce5903", - "point/steady-state-consistency": "9f751df4221317b44d13882c357c21b87e1ddbd1d62a88e3ee9e3dd89d40a5ae", - "point/steady-state-basis": "c06a9eae5142326f1daab8db85b8a2bc8908f9d261ce3446d18b192b277f8177", - "point/point-duration": "d5d109fd65f15b494af5554f82214e4d37adbe32b9407dc91a29ebba5037eafa", - "point/min-completed-samples": "2d88a3c7a939f0075e67cdbdcde7152a19aad7409df42073c33eb309bc497b87", + "point/maximal-engagement": "7ed77d2c82deb62d3d8396eabe191202a61180fa43d9625ad925bcb64da554db", + "point/metric-consistency-accounting": "4bc95657f7b2a46d3813e73a267044d0e4c93e299c02503c6d30e212aecfae5f", "point/metric-consistency-duration": "e8dd3ddb3fd5d339147e4218ed178b20c620f0a8bcc5997ce70babdddf7b556b", - "point/metric-consistency-accounting": "00600517efa73cb889b43fb4c31f7ae7b28de1bb8020da0a9c18e48369b08a03", "point/metric-consistency-output-tokens": "a9464734c55d01306cff0baf8b90aecce9c1052379726f11354cf1cf741461af", "point/metric-consistency-system-tps": "fb2fa8305218403a0bd99cca615f58f802d2dea78b646cc485fcb7b9bc20b1a8", "point/metric-consistency-tpot-p90": "03379bcdce014772308db16a0fa6d3be734d33a742de4c28b8919a7f6debe6ba", - "point/metric-consistency-tps-per-user": "3d33681d535fcfb1173e80ddf4da57b2dfb401c39821f07851a873bb43fa17f0", "point/metric-consistency-tps-per-kw": "a8423347c77fbc0cb9ff347c687f80307ffab6c9ffd07fb2c81203125c6b93db", - "point/agentic-metric-consistency": "2c08e062819cfb8f397938cc0c96030dcb34fb379e0681a0e79d211e6c768731", + "point/metric-consistency-tps-per-user": "3d33681d535fcfb1173e80ddf4da57b2dfb401c39821f07851a873bb43fa17f0", + "point/min-completed-samples": "2d88a3c7a939f0075e67cdbdcde7152a19aad7409df42073c33eb309bc497b87", + "point/nodes-used": "3b0164f362ed651e2feb2ed00d2ae790c974ac74a3e364008699f7a1166be2e0", + "point/offline-declared": "71281f294529c5665bde54020b1f8a5e204b9ed5735f9a135458eb2cae3c6a5a", + "point/point-config-valid": "4bb6c62e68b0748f1a1b47110f0083029d0024958d17f4b8db38af21aed9dfaf", + "point/point-dirname-concurrency": "2296b404dacb1a6f15106be0dc2a6acea4d519d54e44c486e35c6ac78ec066fc", + "point/point-disclosure-complete": "218ae026f7fdec937b0161af67519e60698a087c52eac59e579ae77203b254be", + "point/point-duration": "d5d109fd65f15b494af5554f82214e4d37adbe32b9407dc91a29ebba5037eafa", + "point/point-power": "6e75d380e17c6198e37ade0ac1ebd4598fc57c367a7994ff9543a75cb2106961", + "point/point-rules-skipped": "d469f8eff6daf0307fc7309f0b2d4db6b05f0649f649ec56a5017efaa67d0021", + "point/region-declared": "e4340e52fed15f4a3dee74b73086f6688344057adddc45d737fb0bb88475719c", + "point/region-placement": "283c684b2b3276f211af1a45f78f07702e804560a05a3e9993a05c58ad1c8283", + "point/result-file-valid": "e5cebe2ee4158e9630ada483b0ac6596c3ee58608313a152ba8a460337f9f5f8", + "point/result-summary-present": "9eaf56fa9c6a152f94563d2d8555a5435b887c7db9d3d9c5769c2f87c5b971d8", "point/seed-config-legacy": "71b0b9e90407e7d0a8283abe23584727a267d19baa078f892c21802fb87160c1", - "point/seed-set-membership": "8d28659d6b0be81fe43d400a75aedf9bad54f68fd7ee8985ffe98055e536c338", "point/seed-runtime-match": "530f81fec40bc8738b7b4f28268de0aad7a74e37c61d03e0391472b2cbae6796", "point/seed-set-adoption": "b4f9895bcb58c00c69add9cdb33058c251ec3a1d208a10c5fc01693ac33684a5", - "point/approved-drafter": "eafdddb450939724ddfe0f45026caf2b18ada8f6e599e578a342e19ab3836b0c", - "point/drafter-approval-lead-time": "d4d43369592c41a488033384eacc4cc7c3cdfac4c715d981959439d6f611e066", - "point/nodes-used": "3b0164f362ed651e2feb2ed00d2ae790c974ac74a3e364008699f7a1166be2e0", - "point/maximal-engagement": "7ed77d2c82deb62d3d8396eabe191202a61180fa43d9625ad925bcb64da554db", - "point/point-rules-skipped": "d469f8eff6daf0307fc7309f0b2d4db6b05f0649f649ec56a5017efaa67d0021", - "diagnostics/seed-set-registry": "7670c90fb34c6459fc0a1808c999337db09d7e7d9a95283521d9235cddd4d647", - "diagnostics/drafter-list-registry": "5b753c1c9d92828a13bbe866841833612e283698771a6c6992b7c717532c6904" - } + "point/seed-set-membership": "8d28659d6b0be81fe43d400a75aedf9bad54f68fd7ee8985ffe98055e536c338", + "point/shared-path-resolution": "4f6901ad243563cad240c70a7f0afed406639e5ee6d2f5ef4ea15b3ebc88ff14", + "point/spec-decode-head-approval-lead-time": "9921be831577a77cf77c54bca503520fb2ff8e9cc5e1abc431df2f3bef5e67b8", + "point/steady-state-basis": "c06a9eae5142326f1daab8db85b8a2bc8908f9d261ce3446d18b192b277f8177", + "point/steady-state-consistency": "9f751df4221317b44d13882c357c21b87e1ddbd1d62a88e3ee9e3dd89d40a5ae", + "point/steady-state-valid": "39699685e3a49ff6db27ee6c02405878ef2362262f03f37088ac2a4a42ce5903", + "point/streaming-config": "6b7d56890aed2d66ff24c2c1ee8a9b1b8cd4d465f94e8dc800b76ea9632f2fd9", + "point/submission-flags": "6fd56145219a078dd947d79bd1a5587e9c45da258085ea0e0145d3d75e7330a9", + "point/swebench-instance-count": "84477a7f4dc7e0da91fabdf4e7ec73cbd886194eeadb3895136e4594980304fc", + "point/swebench-template": "664ef45ae9124f181b8b1e947e25f29b6567ebb09c4ea192052c50678435dc15", + "point/system-description-present": "856c0e732cba243a1bb153ab39176653a0d3a33fd33958095e564cd0e529f785", + "point/system-description-valid": "b53aee4801f00277cece609d210b3ce4496a257fe0d1d6a3ea2bda7b24b80c1d", + "point/target-cohort": "6855eb240be6164648dccd441741230bca1783680c0b53a67cdf6b54e5466eba", + "point/trajectories-to-issue": "323a70bf8bca223b7f926afb53bfeacb519f59cc51949484b3643642bf2e79ab", + "point/warmup-logs-retained": "d63e4b1891824a0834adca73b95a192bbf3685536bbb4161f21d1a4cc72e7de4", + "point/warmup-present": "ade8ebb85859a4824b300ce75598921791c5b7f9cd9a9569d4729f27404aced1", + "point/warmup-salt": "091a57e9035d9a7f98e64cec9429bb5ed5b5a5a1b1f3b4f8510e191b7bc111f8", + "submission/accuracy-present": "2f0dc1d59739717847bcb1519c78d6f2bbd871e9a1e666e6daa99b31f0a0824c", + "submission/path-exists": "10d1f797fff02890ddd7e3473732b96c44520d23952f9b7493fc576f6e5bf6a3", + "submission/required-dir": "17674a08b2897de6897d22270998e999d0ff8ac118a35d263555604c727a3f9d", + "submission/src-dir": "a5b4a62887d8636f2f758a717100b68bfe31f3107648536236e990569f971cfa", + "submission/src-readme": "5754b3df0d5389985d5a476d630274d9fa1143e6cfa04be8616e119a55b5ac3b", + "submission/system-results-dir": "72456038d0d00ed12a7e3d58eda518b86b69aea7b019fad32b13a8310359704c", + "system/benchmark-model-dir": "517f85dedefe201e639a74d83c581cd9efa481731848761d9ade76df7326bb3e", + "system/power-descriptor": "7b3b36310067d0772fb58c927e219d68c73881c25f62637fcb98498b6e9639a7", + "system/power-estimated": "d561adae981297c84fe746712f3e4008857e2e9c947c9d69f41e6595c3409fd3", + "system/system-description-across-curves": "6ff9a22fb2d7bc71fcdb91c7a13c8bee6bec0f8cfaefc5c5f1ad80c901c23f43", + "system/system-name-consistency": "7ef9d2b5a293776dd30c1fea663e1b401678c75f826d89743cf0706a628784e6" + }, + "version": "2026-10-C1" } diff --git a/tests/unit/validation/test_engine.py b/tests/unit/validation/test_engine.py index 741bc4114..ec6627d8d 100644 --- a/tests/unit/validation/test_engine.py +++ b/tests/unit/validation/test_engine.py @@ -9,6 +9,7 @@ import pytest import yaml +from pydantic import ValidationError from inference_endpoint.validation import ( Conditions, @@ -212,19 +213,10 @@ def test_region_basis_uses_selected_collection_members(tmp_path): @pytest.mark.parametrize("rule_id", ["region-placement", "offline-point-present"]) -def test_unknown_collection_operation_blocks(rule_id): - policy = policy_only("region-basis", rule_id) - rules = tuple( - replace( - rule, requirements=freeze({**rule.requirements, "operation": "unsupported"}) - ) - if rule.id == rule_id - else rule - for rule in policy.checks - ) - report = execute_submission(FIXTURE, replace(policy, checks=rules)) - findings = [f for f in report.results if f.rule == rule_id] - assert findings and all(f.key == "blocked" for f in findings) +def test_unknown_collection_operation_is_rejected(rule_id): + rule = next(rule for rule in policy_only(rule_id).checks if rule.id == rule_id) + with pytest.raises(ValidationError, match="operation"): + rule.requirements.with_updates({"operation": "unsupported"}) def test_offline_uses_effective_numeric_concurrency_floor(tmp_path): @@ -237,8 +229,8 @@ def test_offline_uses_effective_numeric_concurrency_floor(tmp_path): policy = policy_only("offline-ordering") rule = replace( policy.checks[0], - requirements=freeze( - {**policy.checks[0].requirements, "concurrency_floor": 10**12} + requirements=policy.checks[0].requirements.with_updates( + {"concurrency_floor": 10**12} ), ) report = execute_submission(submission, replace(policy, checks=(rule,))) @@ -250,8 +242,8 @@ def test_elected_offline_uses_effective_required_concurrency(): policy = policy_only("offline-point-present") rule = replace( policy.checks[0], - requirements=freeze( - {**policy.checks[0].requirements, "elected_must_equal": 10**12} + requirements=policy.checks[0].requirements.with_updates( + {"elected_must_equal": 10**12} ), ) assert not execute_submission(FIXTURE, replace(policy, checks=(rule,))).passed @@ -292,7 +284,7 @@ def test_conflicting_region_basis_partitions_block_explicitly(): @pytest.mark.parametrize( - "option", ["seed_sets_path", "approved_sped_decode_heads_path"] + "option", ["seed_sets_path", "approved_spec_decode_heads_path"] ) def test_api_rejects_catalog_overrides(option, tmp_path): with pytest.raises(TypeError, match="unexpected keyword argument"): @@ -302,15 +294,15 @@ def test_api_rejects_catalog_overrides(option, tmp_path): def test_catalog_environment_cannot_replace_validation_approvals(tmp_path, monkeypatch): monkeypatch.setenv("MLPERF_ENDPOINTS_SEED_SETS", str(tmp_path / "seeds.yaml")) monkeypatch.setenv( - "MLPERF_ENDPOINTS_APPROVED_DRAFTERS", str(tmp_path / "heads.yaml") + "MLPERF_ENDPOINTS_APPROVED_SPEC_DECODE_HEADS", str(tmp_path / "heads.yaml") ) policy = load_policy(bundled_policy_path()) artifacts = load_artifacts(tmp_path, policy) assert "seed_sets" in artifacts.catalogs.values assert not artifacts.catalogs.errors assert ( - artifacts.catalogs.values["approved_sped_decode_heads"] - == policy.catalogs["approved_sped_decode_heads"] + artifacts.catalogs.values["approved_spec_decode_heads"] + == policy.catalogs["approved_spec_decode_heads"] ) diff --git a/tests/unit/validation/test_final_review_regressions.py b/tests/unit/validation/test_final_review_regressions.py new file mode 100644 index 000000000..e3f0fb163 --- /dev/null +++ b/tests/unit/validation/test_final_review_regressions.py @@ -0,0 +1,354 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Submission boundary and metric calculation regressions.""" + +import json +import shutil +from dataclasses import replace +from pathlib import Path + +import pytest +import yaml +from pydantic import ValidationError + +from inference_endpoint.validation import ( + bundled_policy_path, + load_policy, + validate_submission, +) +from inference_endpoint.validation.evidence.loaders import ( + MAX_ARTIFACT_DEPTH, + ParsedArtifact, + supplied_fields, +) +from inference_endpoint.validation.evidence.point_config import PointConfig +from inference_endpoint.validation.evidence.point_summary import PointSummary +from inference_endpoint.validation.operations import SeedOperation +from inference_endpoint.validation.schemas.requirements_v1 import ( + SeedBindingRequirements, +) + +pytestmark = pytest.mark.unit +FIXTURE = ( + Path(__file__).resolve().parents[2] + / "fixtures/validation/submissions/valid_standardized" +) + + +def selected_policy(*ids): + policy = load_policy(bundled_policy_path()) + return replace( + policy, checks=tuple(rule for rule in policy.checks if rule.id in ids) + ) + + +def write_summary(root, summary): + point = root / "results/system/gpt-oss-120b/r16" + point.mkdir(parents=True) + (point / "result_summary.json").write_text( + json.dumps({"n_samples_completed": 100, "duration_ns": 1e9, **summary}) + ) + return point + + +def test_system_names_are_compared_within_each_system(tmp_path): + shutil.copytree(FIXTURE, tmp_path / "submission") + root = tmp_path / "submission" + systems = root / "results" + original = next(systems.iterdir()) + second = systems / "system_second" + shutil.copytree(original, second) + for path in second.glob("*/r*/system_desc.json"): + data = json.loads(path.read_text()) + data["system_name"] = "system_second" + path.write_text(json.dumps(data)) + policy = selected_policy("system-name-consistency") + report = validate_submission(root, policy=policy) + assert len(report.results) == 2 + assert report.passed + path = next(second.glob("*/r*/system_desc.json")) + data = json.loads(path.read_text()) + data["system_name"] = "inconsistent" + path.write_text(json.dumps(data)) + report = validate_submission(root, policy=policy) + assert len(report.errors) == 1 + assert report.errors[0].path == second + + +@pytest.mark.parametrize( + "explicit,failed,expected", + [ + ({}, 0, 100), + ({}, 1, None), + ( + {"output_tokens_per_turn_total": 200, "e2e_turn_time_seconds_total": 1}, + 1, + 200, + ), + ({"output_tokens_per_turn_total": 200}, 0, None), + ({"e2e_turn_time_seconds_total": 1}, 0, None), + ( + {"output_tokens_per_turn_total": 200, "e2e_turn_time_seconds_total": 0}, + 0, + None, + ), + ], +) +def test_interactivity_derivation_preserves_population_and_precedence( + explicit, failed, expected +): + summary = PointSummary.model_validate( + { + "n_samples_completed": 100, + "n_samples_failed": failed, + "duration_ns": 1e9, + "output_sequence_lengths": {"total": 1000}, + "latency": {"total": 1e10}, + **explicit, + } + ) + assert summary.e2e_avg_interactivity == expected + + +def test_client_latency_totals_execute_interactivity_check(tmp_path): + write_summary( + tmp_path, + { + "output_sequence_lengths": {"total": 1000}, + "latency": {"total": 1e10}, + "e2e_avg_interactivity": 100, + }, + ) + report = validate_submission( + tmp_path, policy=selected_policy("agentic-metric-consistency") + ) + assert len(report.results) == 1 + assert report.passed + + +@pytest.mark.parametrize( + "field", + ["duration_ns", "output_tokens_per_turn_total", "e2e_turn_time_seconds_total"], +) +def test_nonfinite_summary_scalars_are_invalid(field): + with pytest.raises(ValidationError, match="finite"): + PointSummary.model_validate( + {"n_samples_completed": 100, "duration_ns": 1e9, field: float("inf")} + ) + + +@pytest.mark.parametrize("stored", [None, 1]) +def test_finite_inputs_with_overflowing_derived_throughput_fail(tmp_path, stored): + data = {"duration_ns": 1e-299, "output_sequence_lengths": {"total": 1e100}} + if stored is not None: + data["system_tps"] = stored + write_summary(tmp_path, data) + report = validate_submission( + tmp_path, policy=selected_policy("metric-consistency-system-tps") + ) + assert len(report.errors) == 1 + assert "not finite" in report.errors[0].message + + +def test_overflowing_json_number_produces_invalid_artifact(tmp_path): + point = write_summary(tmp_path, {}) + (point / "result_summary.json").write_text( + '{"n_samples_completed": 13368, "duration_ns": 1e9, "output_sequence_lengths": {"total": 1e309}}' + ) + report = validate_submission(tmp_path, policy=selected_policy("result-file-valid")) + assert report.errors + assert all(result.rule == "result-file-valid" for result in report.errors) + assert any("finite" in result.message for result in report.errors) + + +def test_recursive_yaml_alias_is_invalid_artifact(tmp_path): + point = write_summary(tmp_path, {}) + (point / "point.yaml").write_text( + "concurrency: 16\nruntime_settings: {runtime: {}}\ncycle: &loop {self: *loop}\n" + ) + report = validate_submission(tmp_path, policy=selected_policy("point-config-valid")) + assert len(report.errors) == 1 + assert "recursive alias" in report.errors[0].message + + +def test_shared_alias_records_each_path(): + shared = {"leaf": 1} + assert supplied_fields({"a": shared, "b": shared}) == frozenset( + {"a", "a.leaf", "b", "b.leaf"} + ) + + +def test_cycle_through_list_is_invalid_artifact(tmp_path): + cycle = [] + cycle.append(cycle) + parsed = ParsedArtifact.from_json( + {"unused": cycle}, PointConfig, tmp_path, "point-config-valid" + ) + assert parsed.value is None + assert "recursive alias" in parsed.errors[0].message + + +def test_excessive_nesting_is_invalid_artifact(tmp_path): + value = {} + for _ in range(MAX_ARTIFACT_DEPTH + 2): + value = {"child": value} + parsed = ParsedArtifact.from_json( + value, PointConfig, tmp_path, "point-config-valid" + ) + assert parsed.value is None + assert "nesting exceeds" in parsed.errors[0].message + + +@pytest.mark.parametrize( + "issued,completed,failed,passed", + [ + (100, 99, 1, True), + (100, 100, 1, False), + (100, 100, 0, True), + (100, 100, None, True), + ], +) +def test_sample_accounting_includes_failures( + tmp_path, issued, completed, failed, passed +): + summary = {"n_samples_issued": issued, "n_samples_completed": completed} + if failed is not None: + summary["n_samples_failed"] = failed + write_summary(tmp_path, summary) + report = validate_submission( + tmp_path, policy=selected_policy("metric-consistency-accounting") + ) + assert len(report.results) == 1 + assert report.passed is passed + + +@pytest.mark.parametrize("override", [False, True]) +def test_unknown_policy_mode_is_rejected_before_selection(tmp_path, override): + directory = tmp_path / "2026-10-C1" + shutil.copytree(bundled_policy_path(), directory) + path = directory / ("catalog.yaml" if override else "curve_checks.yaml") + document = yaml.safe_load(path.read_text()) + if override: + document["enrollment"]["overrides"].append( + { + "id": "bad-weights", + "rule": "accuracy-gate", + "reason": "test", + "applies_to": {"models": ["gpt-oss-120b"]}, + "requirements": {"on_missing_weights": "not_an_implemented_mode"}, + } + ) + else: + document["checks"]["accuracy-gate"]["on_missing_weights"] = ( + "not_an_implemented_mode" + ) + path.write_text(yaml.safe_dump(document, sort_keys=False)) + with pytest.raises(ValueError, match="on_missing_weights"): + load_policy(directory) + + +def test_requirement_sequences_and_aliases_are_immutable(): + aliases = {"scheduler_random_seed": "scheduler_rng_seed"} + requirements = SeedBindingRequirements( + operation=SeedOperation.LEGACY_NAMES, aliases=aliases + ) + aliases["scheduler_random_seed"] = "changed" + assert requirements.aliases["scheduler_random_seed"] == "scheduler_rng_seed" + with pytest.raises(TypeError): + requirements.aliases["scheduler_random_seed"] = "changed" + assert requirements.with_updates({}).wire() == requirements.wire() + + +@pytest.mark.parametrize( + "operands,issued,passed", + [ + ( + [ + { + "kind": "sum", + "operands": [ + {"kind": "constant", "value": 10}, + {"kind": "constant", "value": 20}, + ], + }, + {"kind": "field", "field": "result_summary.n_samples_completed"}, + ], + 130, + True, + ), + ( + [ + {"kind": "field", "field": "result_summary.tps_per_user", "default": 7}, + {"kind": "field", "field": "result_summary.n_samples_completed"}, + ], + 107, + True, + ), + ( + [ + {"kind": "field", "field": "result_summary.tps_per_user"}, + {"kind": "field", "field": "result_summary.n_samples_completed"}, + ], + 100, + False, + ), + ( + [ + {"kind": "constant", "value": 1e308}, + {"kind": "constant", "value": 1e308}, + ], + 100, + False, + ), + ], +) +def test_sum_operands_support_nesting_defaults_and_missing_values( + tmp_path, operands, issued, passed +): + policy_dir = tmp_path / "2026-10-C1" + shutil.copytree(bundled_policy_path(), policy_dir) + path = policy_dir / "point_checks.yaml" + document = yaml.safe_load(path.read_text()) + document["checks"]["metric-consistency-accounting"]["left"] = { + "kind": "sum", + "operands": operands, + } + path.write_text(yaml.safe_dump(document, sort_keys=False)) + policy = load_policy(policy_dir) + policy = replace( + policy, + checks=tuple( + rule for rule in policy.checks if rule.id == "metric-consistency-accounting" + ), + ) + write_summary(tmp_path, {"n_samples_issued": issued}) + report = validate_submission(tmp_path, policy=policy) + assert len(report.results) == 1 + assert report.passed is passed + assert all( + "Evaluation could not complete" not in result.message + for result in report.results + ) + + +@pytest.mark.parametrize("duration", [1e-320, 1e-299]) +def test_tiny_positive_durations_never_crash_derivation(tmp_path, duration): + write_summary( + tmp_path, + { + "duration_ns": duration, + "output_sequence_lengths": {"total": 1e100}, + "tpot": {"percentiles": {"90": duration}}, + }, + ) + report = validate_submission( + tmp_path, + policy=selected_policy( + "metric-consistency-system-tps", "metric-consistency-tps-per-user" + ), + ) + assert report.errors + assert all( + "Evaluation could not complete" not in result.message + for result in report.results + ) diff --git a/tests/unit/validation/test_loader.py b/tests/unit/validation/test_loader.py index 396c0c434..c711fa814 100644 --- a/tests/unit/validation/test_loader.py +++ b/tests/unit/validation/test_loader.py @@ -9,6 +9,7 @@ import pytest import yaml +from pydantic import ValidationError from inference_endpoint.validation import bundled_policy_path, load_policy from inference_endpoint.validation.schemas import BundleParser, parser_for @@ -61,8 +62,10 @@ def test_policy_is_immutable_and_preserves_source_hashes(policy_dir): policy = load_policy(policy_dir) with pytest.raises(TypeError): policy.catalogs["datasets"]["aime25"]["sample_count"] = 1 - with pytest.raises(TypeError): - policy.checks[0].requirements["anything"] = 1 + requirements = policy.checks[0].requirements + field = next(iter(type(requirements).model_fields)) + with pytest.raises(ValidationError, match="frozen"): + setattr(requirements, field, getattr(requirements, field)) for file, digest in policy.source_digests.items(): assert ( digest == hashlib.sha256((policy_dir / file.value).read_bytes()).hexdigest() @@ -197,27 +200,27 @@ def duplicate(data): def test_optional_offline_and_region_declarations_are_checked_only_when_present(): policy = load_policy(bundled_policy_path()) rules = {rule.id: rule for rule in policy.checks} - assert rules["offline-declared"].requirements["required"] is False - assert rules["region-declared"].requirements["required"] is False + assert rules["offline-declared"].requirements.required is False + assert rules["region-declared"].requirements.required is False def test_decode_head_can_record_its_publication_cohort(policy_dir): catalog = policy_dir / PolicyFile.CATALOG.value modify( catalog, - lambda data: data["catalogs"]["approved_sped_decode_heads"][0].update( + lambda data: data["catalogs"]["approved_spec_decode_heads"][0].update( approved_cohort="2026-06-C0" ), ) assert ( - load_policy(policy_dir).catalogs["approved_sped_decode_heads"][0][ + load_policy(policy_dir).catalogs["approved_spec_decode_heads"][0][ "approved_cohort" ] == "2026-06-C0" ) modify( catalog, - lambda data: data["catalogs"]["approved_sped_decode_heads"][0].update( + lambda data: data["catalogs"]["approved_spec_decode_heads"][0].update( approved_cohort="2026-06-C9" ), ) diff --git a/tests/unit/validation/test_planner.py b/tests/unit/validation/test_planner.py index cb6cd725c..7fb77fb49 100644 --- a/tests/unit/validation/test_planner.py +++ b/tests/unit/validation/test_planner.py @@ -51,7 +51,7 @@ def decisions(policy, context): def test_patterns_select_model_family_checks(policy): agentic = decisions(policy, point()) assert agentic["trajectories-to-issue"].decision is Decision.READY - assert agentic["approved-drafter"].decision is Decision.EXCLUDED + assert agentic["approved-spec-decode-head"].decision is Decision.EXCLUDED single = decisions( policy, point(model_id="llama3_1-8b", load_pattern="concurrency") ) @@ -184,12 +184,12 @@ def test_model_override_changes_only_selected_context(tmp_path): policy = load_policy(directory) check = decisions(policy, point())["trajectories-to-issue"] assert check.override == "kimi-count-exception" - assert check.rule.requirements["operator"] == "equal" + assert check.rule.requirements.operator == "equal" other = decisions(policy, point(model_id="deepseek-v4_1-flash"))[ "trajectories-to-issue" ] assert other.override is None - assert other.rule.requirements["operator"] == "positive_multiple" + assert other.rule.requirements.operator == "positive_multiple" data["enrollment"]["overrides"].append({**override, "id": "overlapping"}) path.write_text(yaml.safe_dump(data, sort_keys=False)) with pytest.raises(ValueError, match="Overlapping overrides"): diff --git a/tests/unit/validation/test_policy_contracts.py b/tests/unit/validation/test_policy_contracts.py index 0d72883f7..506b7b88b 100644 --- a/tests/unit/validation/test_policy_contracts.py +++ b/tests/unit/validation/test_policy_contracts.py @@ -62,8 +62,19 @@ def test_accuracy_nested_scores_are_owned(): assert parsed.root["aime25"]["extras"]["source"] == "test" -@pytest.mark.parametrize("field", ["speculative_decoding", "drafter"]) -@pytest.mark.parametrize("declaration", [{}, {"target_checksum": "declared-checksum"}]) +@pytest.mark.parametrize( + "field", ["speculative_decoding", "spec_decode_head", "drafter"] +) +@pytest.mark.parametrize( + "declaration", + [ + {}, + { + "target_checksum": "git-sha1:" + "a" * 40, + "configuration": {"exit_layer": 24}, + }, + ], +) def test_supplied_decode_head_keeps_approval_checks_selected( tmp_path, field, declaration ): @@ -78,15 +89,15 @@ def test_supplied_decode_head_keeps_approval_checks_selected( {}, ) point = PointArtifacts.from_evidence( - point.path, point.evidence, frozenset({EvidenceKey.APPROVED_SPED_DECODE_HEADS}) + point.path, point.evidence, frozenset({EvidenceKey.APPROVED_SPEC_DECODE_HEADS}) ) policy = load_policy(bundled_policy_path()) checks = { check.rule.id: check for check in plan_checks(policy, [point.context]).checks } assert point.context.speculative_decoding is True - assert checks["approved-drafter"].decision is Decision.READY - assert checks["drafter-approval-lead-time"].decision is Decision.READY + assert checks["approved-spec-decode-head"].decision is Decision.READY + assert checks["spec-decode-head-approval-lead-time"].decision is Decision.READY @pytest.fixture @@ -194,19 +205,20 @@ def test_no_decode_head_keeps_approval_checks_excluded(tmp_path, declaration): ).checks } assert point.context.speculative_decoding is False - assert checks["approved-drafter"].decision is Decision.EXCLUDED + assert checks["approved-spec-decode-head"].decision is Decision.EXCLUDED -def test_incomplete_canonical_head_does_not_fall_back_to_alias(tmp_path): +@pytest.mark.parametrize("alias", ["spec_decode_head", "drafter"]) +def test_incomplete_canonical_head_does_not_fall_back_to_alias(tmp_path, alias): policy = load_policy(bundled_policy_path()) - approved = policy.catalogs["approved_sped_decode_heads"][0] + approved = policy.catalogs["approved_spec_decode_heads"][0] point = tmp_path / "results" / "system" / approved["model"] / "r16" point.mkdir(parents=True) config = { "concurrency": 16, "runtime_settings": {"runtime": {}}, - "speculative_decoding": {"target_checksum": "declared"}, - "drafter": { + "speculative_decoding": {}, + alias: { "repository": approved["repository"], "revision": approved["revision"], }, @@ -214,7 +226,7 @@ def test_incomplete_canonical_head_does_not_fall_back_to_alias(tmp_path): (point / "point.yaml").write_text(json.dumps(config)) report = validate_submission(tmp_path, policy=policy) assert any( - result.rule == "approved-drafter" + result.rule == "approved-spec-decode-head" and not result.passed and "approved identity" in result.message for result in report.results diff --git a/tests/unit/validation/test_review_regressions.py b/tests/unit/validation/test_review_regressions.py index c38c58dab..e474d75cf 100644 --- a/tests/unit/validation/test_review_regressions.py +++ b/tests/unit/validation/test_review_regressions.py @@ -6,6 +6,7 @@ import pytest import yaml +from pydantic import ValidationError from inference_endpoint.validation import ( Context, @@ -165,14 +166,14 @@ def test_enabled_curve_overrides_keep_distinct_immutable_requirements(tmp_path): if check.rule.id == "offline-ordering" ] assert { - (check.override, check.rule.requirements["throughput_minimum_multiplier"]) + (check.override, check.rule.requirements.throughput_minimum_multiplier) for check in checks } == {("max_throughput", 0.9), ("agentic_inference", 0.8)} assert all(check.decision is Decision.READY for check in checks) - with pytest.raises(TypeError): - checks[0].rule.requirements["throughput_minimum_multiplier"] = 1 + with pytest.raises(ValidationError, match="frozen"): + checks[0].rule.requirements.throughput_minimum_multiplier = 1 original = next(rule for rule in policy.checks if rule.id == "offline-ordering") - assert original.requirements["throughput_minimum_multiplier"] == 0.98 + assert original.requirements.throughput_minimum_multiplier == 0.98 def test_overlapping_overrides_are_detected_on_each_curve_member(tmp_path): diff --git a/tests/unit/validation/test_spec_decode_heads.py b/tests/unit/validation/test_spec_decode_heads.py new file mode 100644 index 000000000..2d5aad99d --- /dev/null +++ b/tests/unit/validation/test_spec_decode_heads.py @@ -0,0 +1,409 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""SpecDecodeHead names and accepted declaration forms preserve approval checks.""" + +import json +import shutil +from dataclasses import replace + +import pytest +import yaml +from pydantic import ValidationError + +from inference_endpoint.validation import ( + bundled_policy_path, + load_policy, + validate_submission, +) +from inference_endpoint.validation.evidence.point_config import ( + PointConfig, + SpecDecodeHeadDeclaration, +) +from inference_endpoint.validation.schemas.values_v1 import Catalogs + +pytestmark = pytest.mark.unit + + +@pytest.mark.parametrize( + "field", ["speculative_decoding", "spec_decode_head", "drafter"] +) +@pytest.mark.parametrize("approved", [True, False]) +def test_spec_decode_head_declarations_select_and_run_approval( + tmp_path, field, approved +): + policy = load_policy(bundled_policy_path()) + head = policy.catalogs["approved_spec_decode_heads"][0] + declaration = { + "repository": head["repository"], + "revision": head["revision"] if approved else "0" * 40, + } + point = tmp_path / "results" / "system" / head["model"] / "r16" + point.mkdir(parents=True) + (point / "point.yaml").write_text( + json.dumps( + { + "concurrency": 16, + "runtime_settings": {"runtime": {}}, + field: declaration, + } + ) + ) + selected = replace( + policy, + checks=tuple( + rule for rule in policy.checks if rule.id == "approved-spec-decode-head" + ), + ) + report = validate_submission(tmp_path, policy=selected) + assert len(report.results) == 1 + assert report.passed is approved + + +@pytest.mark.parametrize("canonical", [None, {}, {"repository": "different"}]) +def test_conflicting_spec_decode_head_aliases_are_rejected(canonical): + with pytest.raises(ValidationError, match="input alias must agree"): + PointConfig.model_validate( + { + "concurrency": 16, + "runtime_settings": {"runtime": {}}, + "spec_decode_head": canonical, + "drafter": {"repository": "head", "revision": "a" * 40}, + } + ) + + +def test_spec_decode_head_input_alias_serializes_with_canonical_name(): + parsed = PointConfig.model_validate( + { + "concurrency": 16, + "runtime_settings": {"runtime": {}}, + "drafter": {"repository": "head", "revision": "a" * 40}, + } + ) + payload = parsed.model_dump(mode="json", exclude_unset=True) + assert payload["spec_decode_head"] == {"repository": "head", "revision": "a" * 40} + assert "drafter" not in payload + + +@pytest.mark.parametrize("head_index", range(6)) +@pytest.mark.parametrize( + "target,passed", [("2026-10-C0", False), ("2026-10-C1", True), ("2026-11-C0", True)] +) +def test_published_weight_checksums_and_approval_lead_time( + tmp_path, head_index, target, passed +): + policy = load_policy(bundled_policy_path()) + head = policy.catalogs["approved_spec_decode_heads"][head_index] + assert head["approved_cohort"] == "2026-09-C1" + point = tmp_path / "results/system" / head["model"] / "r16" + point.mkdir(parents=True) + (point / "point.yaml").write_text( + json.dumps( + { + "concurrency": 16, + "runtime_settings": {"runtime": {}}, + "target_cohort": target, + "speculative_decoding": { + "weight_checksum": "git-sha1:" + head["revision"] + }, + } + ) + ) + selected = replace( + policy, + checks=tuple( + rule + for rule in policy.checks + if rule.id + in { + "approved-spec-decode-head", + "spec-decode-head-approval-lead-time", + } + ), + ) + report = validate_submission(tmp_path, policy=selected) + assert len(report.results) == 2 + assert next( + r for r in report.results if r.rule == "approved-spec-decode-head" + ).passed + assert report.passed is passed + assert not report.warnings + + +@pytest.mark.parametrize( + "identity", + [ + {"weight_checksum": "sha256:" + "a" * 64}, + {"weight_checksum": "git-sha1:short"}, + {"weight_checksum": "git-sha1:" + "A" * 40}, + {"weight_checksum": {"revision": "a" * 40}}, + {"weight_checksum": "git-sha1:" + "a" * 40, "revision": "b" * 40}, + ], +) +def test_malformed_or_conflicting_weight_identity_is_rejected(identity): + with pytest.raises(ValidationError): + PointConfig.model_validate( + { + "concurrency": 16, + "runtime_settings": {"runtime": {}}, + "spec_decode_head": identity, + } + ) + + +@pytest.mark.parametrize( + "identity,model", + [ + ({"repository": "wrong"}, None), + ({"revision": "0" * 40}, None), + ({}, "deepseek-v4_1-flash"), + ( + { + "target_checksum": "git-sha1:" + "a" * 40, + "configuration": {"exit_layer": 24}, + }, + None, + ), + ], +) +def test_weight_identity_must_match_model_and_supplied_repository( + tmp_path, identity, model +): + policy = load_policy(bundled_policy_path()) + head = policy.catalogs["approved_spec_decode_heads"][0] + if "target_checksum" not in identity: + checksum_revision = identity.get("revision", head["revision"]) + identity = {"weight_checksum": "git-sha1:" + checksum_revision, **identity} + point = tmp_path / "results/system" / (model or head["model"]) / "r16" + point.mkdir(parents=True) + (point / "point.yaml").write_text( + json.dumps( + { + "concurrency": 16, + "runtime_settings": {"runtime": {}}, + "spec_decode_head": identity, + } + ) + ) + selected = replace( + policy, + checks=tuple( + rule for rule in policy.checks if rule.id == "approved-spec-decode-head" + ), + ) + report = validate_submission(tmp_path, policy=selected) + assert len(report.errors) == 1 + assert not report.passed + + +def test_weight_checksum_round_trip_preserves_string_contract(): + identity = { + "weight_checksum": "git-sha1:" + "a" * 40, + "repository": "head", + "revision": "a" * 40, + } + point = PointConfig.model_validate( + { + "concurrency": 16, + "runtime_settings": {"runtime": {}}, + "spec_decode_head": identity, + } + ) + assert point.spec_decode_head.weight_checksum.revision == "a" * 40 + assert ( + point.model_dump(mode="json", exclude_unset=True)["spec_decode_head"] + == identity + ) + + +@pytest.mark.parametrize("checksum", ["git-sha1:" + "a" * 40, "sha256:" + "a" * 64]) +@pytest.mark.parametrize( + "change,passed", + [ + (None, True), + ("checksum", False), + ("configuration", False), + ("boolean_type", False), + ("number_type", False), + ("model", False), + ("cohort", False), + ], +) +def test_configuration_identity_approval_and_age(tmp_path, checksum, change, passed): + # Synthetic policy entry exercises support without adding an official approval. + policy_dir = tmp_path / "2026-10-C1" + shutil.copytree(bundled_policy_path(), policy_dir) + catalog_path = policy_dir / "catalog.yaml" + document = yaml.safe_load(catalog_path.read_text()) + model = document["catalogs"]["approved_spec_decode_heads"][0]["model"] + configuration = {"exit_layer": 24, "layers": [1, 2], "settings": {"enabled": True}} + document["catalogs"]["approved_spec_decode_heads"].append( + { + "model": model, + "method": "self_speculative", + "target_checksum": checksum, + "configuration": configuration, + "approved_cohort": "2026-09-C1", + } + ) + catalog_path.write_text(yaml.safe_dump(document)) + policy = load_policy(policy_dir) + declaration = {"target_checksum": checksum, "configuration": configuration} + if change == "checksum": + declaration["target_checksum"] = checksum.replace("a", "b") + if change == "configuration": + declaration["configuration"] = {"exit_layer": 12} + if change == "boolean_type": + declaration["configuration"] = {**configuration, "settings": {"enabled": 1}} + if change == "number_type": + declaration["configuration"] = {**configuration, "layers": [1.0, 2]} + if change == "model": + model = "deepseek-v4_1-flash" + point = tmp_path / "submission/results/system" / model / "r16" + point.mkdir(parents=True) + (point / "point.yaml").write_text( + json.dumps( + { + "concurrency": 16, + "runtime_settings": {"runtime": {}}, + "target_cohort": "2026-10-C0" if change == "cohort" else "2026-10-C1", + "speculative_decoding": declaration, + } + ) + ) + selected = replace( + policy, + checks=tuple( + rule + for rule in policy.checks + if rule.id + in { + "approved-spec-decode-head", + "spec-decode-head-approval-lead-time", + } + ), + ) + report = validate_submission(tmp_path / "submission", policy=selected) + assert report.passed is passed + assert not report.warnings + if passed: + assert len(report.results) == 2 + payload = PointConfig.model_validate( + json.loads((point / "point.yaml").read_text()) + ) + assert ( + payload.model_dump(mode="json", exclude_unset=True)["speculative_decoding"] + == declaration + ) + + +@pytest.mark.parametrize( + "identity", + [ + {"target_checksum": "git-sha1:" + "a" * 40}, + {"configuration": {"exit_layer": 24}}, + {"target_checksum": "git-sha1:" + "a" * 40, "configuration": {}}, + {"target_checksum": "invalid", "configuration": {"exit_layer": 24}}, + { + "target_checksum": "git-sha1:" + "a" * 40, + "configuration": {"exit_layer": 24}, + "weight_checksum": "git-sha1:" + "a" * 40, + }, + ], +) +def test_incomplete_or_mixed_configuration_identity_is_rejected(identity): + with pytest.raises(ValidationError): + PointConfig.model_validate( + { + "concurrency": 16, + "runtime_settings": {"runtime": {}}, + "speculative_decoding": identity, + } + ) + + +def test_bundled_catalog_has_no_configuration_based_approvals(): + policy = load_policy(bundled_policy_path()) + assert all( + "target_checksum" not in entry + for entry in policy.catalogs["approved_spec_decode_heads"] + ) + + +@pytest.mark.parametrize( + "change", ["missing_configuration", "empty_configuration", "mixed_identity"] +) +def test_invalid_configuration_catalog_identity_is_rejected(change): + policy = load_policy(bundled_policy_path()) + document = policy.as_monolith()["catalogs"] + entry = { + "model": "kimi-k3", + "method": "self_speculative", + "target_checksum": "sha256:" + "a" * 64, + "configuration": {"exit_layer": 24}, + } + if change == "missing_configuration": + del entry["configuration"] + elif change == "empty_configuration": + entry["configuration"] = {} + else: + entry.update(repository="head", revision="a" * 40) + document["approved_spec_decode_heads"].append(entry) + with pytest.raises(ValidationError): + Catalogs.model_validate(document) + + +@pytest.mark.parametrize( + "declared,approved,expected", + [ + ({"enabled": True}, {"enabled": 1}, False), + ({"enabled": False}, {"enabled": 0}, False), + ({"exit_layer": 1}, {"exit_layer": 1.0}, False), + ( + {"nested": {"layers": [True, {"temperature": False}]}}, + {"nested": {"layers": [1, {"temperature": 0}]}}, + False, + ), + ({"layers": [1, 2]}, {"layers": [2, 1]}, False), + ({"layers": [1, 2]}, {"layers": [1]}, False), + ({"enabled": True}, {"enabled": True, "extra": None}, False), + ({"value": None}, {"value": "null"}, False), + ({"value": float("inf")}, {"value": float("inf")}, False), + ({"value": float("nan")}, {"value": float("nan")}, False), + ( + {"layers": [1, 2.0, True, None, "value"]}, + {"layers": [1, 2.0, True, None, "value"]}, + True, + ), + ( + {"one": 1, "two": {"enabled": False}}, + {"two": {"enabled": False}, "one": 1}, + True, + ), + ], +) +def test_configuration_identity_preserves_json_types(declared, approved, expected): + checksum = "git-sha1:" + "a" * 40 + head = SpecDecodeHeadDeclaration.model_validate( + {"target_checksum": checksum, "configuration": declared} + ) + assert head.matches(target_checksum=checksum, configuration=approved) is expected + + +def test_configuration_identity_aliases_cannot_hide_type_mismatch(): + checksum = "git-sha1:" + "a" * 40 + with pytest.raises(ValidationError, match="input alias must agree"): + PointConfig.model_validate( + { + "concurrency": 16, + "runtime_settings": {"runtime": {}}, + "spec_decode_head": { + "target_checksum": checksum, + "configuration": {"enabled": True}, + }, + "drafter": { + "target_checksum": checksum, + "configuration": {"enabled": 1}, + }, + } + ) diff --git a/tests/unit/validation/test_special_evaluators.py b/tests/unit/validation/test_special_evaluators.py index b9417b09c..2d89cbbbe 100644 --- a/tests/unit/validation/test_special_evaluators.py +++ b/tests/unit/validation/test_special_evaluators.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest +from pydantic import ValidationError from inference_endpoint.validation import ( Conditions, @@ -14,7 +15,6 @@ bundled_policy_path, checks, # noqa: F401 - registers built-in evaluators load_policy, - validate_submission, ) from inference_endpoint.validation.artifacts import ( ArtifactIndex, @@ -89,7 +89,7 @@ def artifacts(): def check_for(artifacts, rule_id, **parameters): rule = next(rule for rule in artifacts.policy.checks if rule.id == rule_id) if parameters: - rule = replace(rule, requirements={**rule.requirements, **parameters}) + rule = replace(rule, requirements=rule.requirements.with_updates(parameters)) point = next(iter(artifacts.index.points.values())) subject = ( point.context @@ -105,6 +105,24 @@ def point(artifacts): return next(iter(artifacts.index.points.values())) +@pytest.mark.parametrize( + "score,passed", [(0.83, True), (83, True), (0.8, False), (80, False)] +) +def test_accuracy_fraction_and_percentage_scores(artifacts, score, passed): + update_accuracy( + point(artifacts), + { + "mlperf_gpt_oss_accuracy": { + "num_samples": 4395, + "score": {"exact_match": score}, + } + }, + ) + results = evaluate(check_for(artifacts, "accuracy-gate"), artifacts) + assert results + assert all(result.passed for result in results) is passed + + def test_accuracy_catalog_threshold_changes_acceptance(artifacts): p = point(artifacts) update_accuracy( @@ -243,26 +261,23 @@ def test_power_catalog_overhead_and_component_defaults(artifacts): assert changed.estimated -def test_unknown_special_operation_is_blocked(artifacts): - results = evaluate( - check_for(artifacts, "warmup-salt", operation="unsupported"), artifacts - ) - assert results[0].key == "blocked" - assert not results[0].passed +def test_unknown_special_operation_is_rejected(artifacts): + with pytest.raises(ValidationError, match="operation"): + check_for(artifacts, "warmup-salt", operation="unsupported") -def test_drafter_identity_requires_repository_and_revision(artifacts): +def test_spec_decode_head_identity_must_be_approved(artifacts): p = point(artifacts) p.context = p.context.model_copy(update={"model_id": "kimi-k3"}) update_evidence( - p, "config", speculative_decoding={"weight_checksum": "same-checksum"} + p, "config", speculative_decoding={"weight_checksum": "git-sha1:" + "0" * 40} ) - results = evaluate(check_for(artifacts, "approved-drafter"), artifacts) + results = evaluate(check_for(artifacts, "approved-spec-decode-head"), artifacts) assert any(not result.passed for result in results) - artifacts.catalogs.values["approved_sped_decode_heads"] = artifacts.policy.catalogs[ - "approved_sped_decode_heads" + artifacts.catalogs.values["approved_spec_decode_heads"] = artifacts.policy.catalogs[ + "approved_spec_decode_heads" ] - approved = artifacts.policy.catalogs["approved_sped_decode_heads"][0] + approved = artifacts.policy.catalogs["approved_spec_decode_heads"][0] update_evidence( p, "config", @@ -273,7 +288,9 @@ def test_drafter_identity_requires_repository_and_revision(artifacts): ) assert all( result.passed - for result in evaluate(check_for(artifacts, "approved-drafter"), artifacts) + for result in evaluate( + check_for(artifacts, "approved-spec-decode-head"), artifacts + ) ) @@ -348,6 +365,7 @@ def test_power_descriptor_override_changes_only_selected_system(two_power_system constants["cooling_overhead"]["air"] = 1.0 artifacts.policy = replace( artifacts.policy, + catalogs={**artifacts.policy.catalogs, "power_high_overhead": constants}, overrides=( Override( "air-overhead", @@ -355,7 +373,7 @@ def test_power_descriptor_override_changes_only_selected_system(two_power_system "System-specific overhead", Conditions(division=["Standardized"]), True, - {"constants": constants}, + {"constants": "catalogs.power_high_overhead"}, ), ), ) @@ -452,26 +470,9 @@ def test_unevaluated_descriptor_provides_no_power_facts( assert other.derived.power_kw == 14.7 -def test_invalid_point_power_policy_returns_blocked_report(two_power_systems): - artifacts = two_power_systems - effective_checks = tuple( - replace( - rule, requirements={**rule.requirements, "round_kw_decimals": "invalid"} - ) - if rule.id == "point-power" - else rule - for rule in artifacts.policy.checks - ) - policy = replace(artifacts.policy, checks=effective_checks) - report = validate_submission(artifacts.root, policy=policy) - assert any( - result.rule == "point-power" and result.key == "blocked" - for result in report.errors - ) - artifacts.policy = policy - prepare(artifacts) - assert all( - point.derived.power_kw is None - and EvidenceKey.POINT_POWER not in point.context.available - for point in artifacts.index.points.values() +def test_invalid_point_power_policy_is_rejected(two_power_systems): + rule = next( + rule for rule in two_power_systems.policy.checks if rule.id == "point-power" ) + with pytest.raises(ValidationError, match="round_kw_decimals"): + rule.requirements.with_updates({"round_kw_decimals": "invalid"}) From f4250d7292de8708400292b912595b81b39e6213 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 13:21:42 -0700 Subject: [PATCH 15/18] test(validation): generate submission artifacts from compact cases --- tests/fixtures/validation/README.md | 40 +- .../validation/cases/invalid_submission.yaml | 250 +++++++++ tests/fixtures/validation/cases/sub_a.yaml | 358 +++++++++++++ tests/fixtures/validation/cases/sub_b.yaml | 358 +++++++++++++ tests/fixtures/validation/cases/sub_c.yaml | 357 +++++++++++++ tests/fixtures/validation/cases/sub_d.yaml | 385 ++++++++++++++ tests/fixtures/validation/cases/sub_e.yaml | 453 +++++++++++++++++ tests/fixtures/validation/cases/sub_f.yaml | 471 +++++++++++++++++ tests/fixtures/validation/cases/sub_g.yaml | 394 ++++++++++++++ tests/fixtures/validation/cases/sub_h.yaml | 394 ++++++++++++++ tests/fixtures/validation/cases/sub_i.yaml | 481 ++++++++++++++++++ tests/fixtures/validation/cases/sub_j.yaml | 480 +++++++++++++++++ .../validation/cases/valid_standardized.yaml | 405 +++++++++++++++ .../invalid_submission/docs/.gitkeep | 0 .../r16/accuracy_results.json | 13 - .../Llama-3_1-8B-Instruct/r16/config.yaml | 1 - .../Llama-3_1-8B-Instruct/r16/point.yaml | 49 -- .../r16/result_summary.json | 29 -- .../r16/system_desc.json | 60 --- .../r38/accuracy_results.json | 13 - .../Llama-3_1-8B-Instruct/r38/config.yaml | 1 - .../Llama-3_1-8B-Instruct/r38/point.yaml | 49 -- .../r38/result_summary.json | 29 -- .../r38/system_desc.json | 60 --- .../r88/accuracy_results.json | 13 - .../Llama-3_1-8B-Instruct/r88/config.yaml | 1 - .../Llama-3_1-8B-Instruct/r88/point.yaml | 50 -- .../r88/result_summary.json | 29 -- .../r88/system_desc.json | 60 --- .../results/system_l/system_power.json | 44 -- .../invalid_submission/src/custom/README.md | 3 - .../submissions/sub_a/docs/.gitkeep | 0 .../gpt-oss-120b/r1024/accuracy_results.json | 11 - .../system_a/gpt-oss-120b/r1024/config.yaml | 1 - .../system_a/gpt-oss-120b/r1024/point.yaml | 49 -- .../gpt-oss-120b/r1024/result_summary.json | 29 -- .../gpt-oss-120b/r1024/system_desc.json | 60 --- .../gpt-oss-120b/r128/accuracy_results.json | 11 - .../system_a/gpt-oss-120b/r128/config.yaml | 1 - .../system_a/gpt-oss-120b/r128/point.yaml | 49 -- .../gpt-oss-120b/r128/result_summary.json | 29 -- .../gpt-oss-120b/r128/system_desc.json | 60 --- .../gpt-oss-120b/r16/accuracy_results.json | 11 - .../system_a/gpt-oss-120b/r16/config.yaml | 1 - .../system_a/gpt-oss-120b/r16/point.yaml | 49 -- .../gpt-oss-120b/r16/result_summary.json | 29 -- .../gpt-oss-120b/r16/system_desc.json | 60 --- .../gpt-oss-120b/r2048/accuracy_results.json | 11 - .../system_a/gpt-oss-120b/r2048/config.yaml | 1 - .../system_a/gpt-oss-120b/r2048/point.yaml | 50 -- .../gpt-oss-120b/r2048/result_summary.json | 29 -- .../gpt-oss-120b/r2048/system_desc.json | 60 --- .../gpt-oss-120b/r4/accuracy_results.json | 11 - .../system_a/gpt-oss-120b/r4/config.yaml | 1 - .../system_a/gpt-oss-120b/r4/point.yaml | 49 -- .../gpt-oss-120b/r4/result_summary.json | 29 -- .../system_a/gpt-oss-120b/r4/system_desc.json | 60 --- .../gpt-oss-120b/r512/accuracy_results.json | 11 - .../system_a/gpt-oss-120b/r512/config.yaml | 1 - .../system_a/gpt-oss-120b/r512/point.yaml | 49 -- .../gpt-oss-120b/r512/result_summary.json | 29 -- .../gpt-oss-120b/r512/system_desc.json | 60 --- .../gpt-oss-120b/r64/accuracy_results.json | 11 - .../system_a/gpt-oss-120b/r64/config.yaml | 1 - .../system_a/gpt-oss-120b/r64/point.yaml | 49 -- .../gpt-oss-120b/r64/result_summary.json | 29 -- .../gpt-oss-120b/r64/system_desc.json | 60 --- .../sub_a/results/system_a/system_power.json | 44 -- .../submissions/sub_a/src/vllm/README.md | 3 - .../submissions/sub_b/docs/.gitkeep | 0 .../gpt-oss-120b/r1024/accuracy_results.json | 11 - .../system_b/gpt-oss-120b/r1024/config.yaml | 1 - .../system_b/gpt-oss-120b/r1024/point.yaml | 49 -- .../gpt-oss-120b/r1024/result_summary.json | 29 -- .../gpt-oss-120b/r1024/system_desc.json | 60 --- .../gpt-oss-120b/r128/accuracy_results.json | 11 - .../system_b/gpt-oss-120b/r128/config.yaml | 1 - .../system_b/gpt-oss-120b/r128/point.yaml | 49 -- .../gpt-oss-120b/r128/result_summary.json | 29 -- .../gpt-oss-120b/r128/system_desc.json | 60 --- .../gpt-oss-120b/r16/accuracy_results.json | 11 - .../system_b/gpt-oss-120b/r16/config.yaml | 1 - .../system_b/gpt-oss-120b/r16/point.yaml | 49 -- .../gpt-oss-120b/r16/result_summary.json | 29 -- .../gpt-oss-120b/r16/system_desc.json | 60 --- .../gpt-oss-120b/r2048/accuracy_results.json | 11 - .../system_b/gpt-oss-120b/r2048/config.yaml | 1 - .../system_b/gpt-oss-120b/r2048/point.yaml | 50 -- .../gpt-oss-120b/r2048/result_summary.json | 29 -- .../gpt-oss-120b/r2048/system_desc.json | 60 --- .../gpt-oss-120b/r4/accuracy_results.json | 11 - .../system_b/gpt-oss-120b/r4/config.yaml | 1 - .../system_b/gpt-oss-120b/r4/point.yaml | 49 -- .../gpt-oss-120b/r4/result_summary.json | 29 -- .../system_b/gpt-oss-120b/r4/system_desc.json | 60 --- .../gpt-oss-120b/r512/accuracy_results.json | 11 - .../system_b/gpt-oss-120b/r512/config.yaml | 1 - .../system_b/gpt-oss-120b/r512/point.yaml | 49 -- .../gpt-oss-120b/r512/result_summary.json | 29 -- .../gpt-oss-120b/r512/system_desc.json | 60 --- .../gpt-oss-120b/r64/accuracy_results.json | 11 - .../system_b/gpt-oss-120b/r64/config.yaml | 1 - .../system_b/gpt-oss-120b/r64/point.yaml | 49 -- .../gpt-oss-120b/r64/result_summary.json | 29 -- .../gpt-oss-120b/r64/system_desc.json | 60 --- .../sub_b/results/system_b/system_power.json | 44 -- .../submissions/sub_b/src/vllm/README.md | 3 - .../submissions/sub_c/docs/.gitkeep | 0 .../r128/accuracy_results.json | 9 - .../qwen3-coder-480b/r128/config.yaml | 1 - .../system_c/qwen3-coder-480b/r128/point.yaml | 49 -- .../qwen3-coder-480b/r128/result_summary.json | 29 -- .../qwen3-coder-480b/r128/system_desc.json | 60 --- .../r16/accuracy_results.json | 9 - .../system_c/qwen3-coder-480b/r16/config.yaml | 1 - .../system_c/qwen3-coder-480b/r16/point.yaml | 49 -- .../qwen3-coder-480b/r16/result_summary.json | 29 -- .../qwen3-coder-480b/r16/system_desc.json | 60 --- .../r256/accuracy_results.json | 9 - .../qwen3-coder-480b/r256/config.yaml | 1 - .../system_c/qwen3-coder-480b/r256/point.yaml | 49 -- .../qwen3-coder-480b/r256/result_summary.json | 29 -- .../qwen3-coder-480b/r256/system_desc.json | 60 --- .../r32/accuracy_results.json | 9 - .../system_c/qwen3-coder-480b/r32/config.yaml | 1 - .../system_c/qwen3-coder-480b/r32/point.yaml | 49 -- .../qwen3-coder-480b/r32/result_summary.json | 29 -- .../qwen3-coder-480b/r32/system_desc.json | 60 --- .../r512/accuracy_results.json | 9 - .../qwen3-coder-480b/r512/config.yaml | 1 - .../system_c/qwen3-coder-480b/r512/point.yaml | 50 -- .../qwen3-coder-480b/r512/result_summary.json | 29 -- .../qwen3-coder-480b/r512/system_desc.json | 60 --- .../r64/accuracy_results.json | 9 - .../system_c/qwen3-coder-480b/r64/config.yaml | 1 - .../system_c/qwen3-coder-480b/r64/point.yaml | 49 -- .../qwen3-coder-480b/r64/result_summary.json | 29 -- .../qwen3-coder-480b/r64/system_desc.json | 60 --- .../qwen3-coder-480b/r8/accuracy_results.json | 9 - .../system_c/qwen3-coder-480b/r8/config.yaml | 1 - .../system_c/qwen3-coder-480b/r8/point.yaml | 49 -- .../qwen3-coder-480b/r8/result_summary.json | 29 -- .../qwen3-coder-480b/r8/system_desc.json | 60 --- .../sub_c/results/system_c/system_power.json | 44 -- .../submissions/sub_c/src/vllm/README.md | 3 - .../submissions/sub_d/docs/.gitkeep | 0 .../r1024/accuracy_results.json | 9 - .../qwen3-coder-480b/r1024/config.yaml | 1 - .../qwen3-coder-480b/r1024/point.yaml | 50 -- .../r1024/result_summary.json | 29 -- .../qwen3-coder-480b/r1024/system_desc.json | 60 --- .../r128/accuracy_results.json | 9 - .../qwen3-coder-480b/r128/config.yaml | 1 - .../system_d/qwen3-coder-480b/r128/point.yaml | 49 -- .../qwen3-coder-480b/r128/result_summary.json | 29 -- .../qwen3-coder-480b/r128/system_desc.json | 60 --- .../r16/accuracy_results.json | 9 - .../system_d/qwen3-coder-480b/r16/config.yaml | 1 - .../system_d/qwen3-coder-480b/r16/point.yaml | 49 -- .../qwen3-coder-480b/r16/result_summary.json | 29 -- .../qwen3-coder-480b/r16/system_desc.json | 60 --- .../r256/accuracy_results.json | 9 - .../qwen3-coder-480b/r256/config.yaml | 1 - .../system_d/qwen3-coder-480b/r256/point.yaml | 49 -- .../qwen3-coder-480b/r256/result_summary.json | 29 -- .../qwen3-coder-480b/r256/system_desc.json | 60 --- .../r32/accuracy_results.json | 9 - .../system_d/qwen3-coder-480b/r32/config.yaml | 1 - .../system_d/qwen3-coder-480b/r32/point.yaml | 49 -- .../qwen3-coder-480b/r32/result_summary.json | 29 -- .../qwen3-coder-480b/r32/system_desc.json | 60 --- .../r512/accuracy_results.json | 9 - .../qwen3-coder-480b/r512/config.yaml | 1 - .../system_d/qwen3-coder-480b/r512/point.yaml | 49 -- .../qwen3-coder-480b/r512/result_summary.json | 29 -- .../qwen3-coder-480b/r512/system_desc.json | 60 --- .../r64/accuracy_results.json | 9 - .../system_d/qwen3-coder-480b/r64/config.yaml | 1 - .../system_d/qwen3-coder-480b/r64/point.yaml | 49 -- .../qwen3-coder-480b/r64/result_summary.json | 29 -- .../qwen3-coder-480b/r64/system_desc.json | 60 --- .../qwen3-coder-480b/r8/accuracy_results.json | 9 - .../system_d/qwen3-coder-480b/r8/config.yaml | 1 - .../system_d/qwen3-coder-480b/r8/point.yaml | 49 -- .../qwen3-coder-480b/r8/result_summary.json | 29 -- .../qwen3-coder-480b/r8/system_desc.json | 60 --- .../sub_d/results/system_d/system_power.json | 44 -- .../submissions/sub_d/src/vllm/README.md | 3 - .../submissions/sub_e/docs/.gitkeep | 0 .../llama3-8b/r1/accuracy_results.json | 15 - .../results/system_e/llama3-8b/r1/config.yaml | 1 - .../results/system_e/llama3-8b/r1/point.yaml | 49 -- .../system_e/llama3-8b/r1/result_summary.json | 29 -- .../system_e/llama3-8b/r1/system_desc.json | 60 --- .../llama3-8b/r1024/accuracy_results.json | 15 - .../system_e/llama3-8b/r1024/config.yaml | 1 - .../system_e/llama3-8b/r1024/point.yaml | 50 -- .../llama3-8b/r1024/result_summary.json | 29 -- .../system_e/llama3-8b/r1024/system_desc.json | 60 --- .../llama3-8b/r128/accuracy_results.json | 15 - .../system_e/llama3-8b/r128/config.yaml | 1 - .../system_e/llama3-8b/r128/point.yaml | 49 -- .../llama3-8b/r128/result_summary.json | 29 -- .../system_e/llama3-8b/r128/system_desc.json | 60 --- .../llama3-8b/r16/accuracy_results.json | 15 - .../system_e/llama3-8b/r16/config.yaml | 1 - .../results/system_e/llama3-8b/r16/point.yaml | 49 -- .../llama3-8b/r16/result_summary.json | 29 -- .../system_e/llama3-8b/r16/system_desc.json | 60 --- .../llama3-8b/r2/accuracy_results.json | 15 - .../results/system_e/llama3-8b/r2/config.yaml | 1 - .../results/system_e/llama3-8b/r2/point.yaml | 49 -- .../system_e/llama3-8b/r2/result_summary.json | 29 -- .../system_e/llama3-8b/r2/system_desc.json | 60 --- .../llama3-8b/r256/accuracy_results.json | 15 - .../system_e/llama3-8b/r256/config.yaml | 1 - .../system_e/llama3-8b/r256/point.yaml | 49 -- .../llama3-8b/r256/result_summary.json | 29 -- .../system_e/llama3-8b/r256/system_desc.json | 60 --- .../llama3-8b/r32/accuracy_results.json | 15 - .../system_e/llama3-8b/r32/config.yaml | 1 - .../results/system_e/llama3-8b/r32/point.yaml | 49 -- .../llama3-8b/r32/result_summary.json | 29 -- .../system_e/llama3-8b/r32/system_desc.json | 60 --- .../llama3-8b/r4/accuracy_results.json | 15 - .../results/system_e/llama3-8b/r4/config.yaml | 1 - .../results/system_e/llama3-8b/r4/point.yaml | 49 -- .../system_e/llama3-8b/r4/result_summary.json | 29 -- .../system_e/llama3-8b/r4/system_desc.json | 60 --- .../llama3-8b/r512/accuracy_results.json | 15 - .../system_e/llama3-8b/r512/config.yaml | 1 - .../system_e/llama3-8b/r512/point.yaml | 49 -- .../llama3-8b/r512/result_summary.json | 29 -- .../system_e/llama3-8b/r512/system_desc.json | 60 --- .../llama3-8b/r64/accuracy_results.json | 15 - .../system_e/llama3-8b/r64/config.yaml | 1 - .../results/system_e/llama3-8b/r64/point.yaml | 49 -- .../llama3-8b/r64/result_summary.json | 29 -- .../system_e/llama3-8b/r64/system_desc.json | 60 --- .../llama3-8b/r8/accuracy_results.json | 15 - .../results/system_e/llama3-8b/r8/config.yaml | 1 - .../results/system_e/llama3-8b/r8/point.yaml | 49 -- .../system_e/llama3-8b/r8/result_summary.json | 29 -- .../system_e/llama3-8b/r8/system_desc.json | 60 --- .../sub_e/results/system_e/system_power.json | 44 -- .../submissions/sub_e/src/vllm/README.md | 3 - .../submissions/sub_f/docs/.gitkeep | 0 .../llama3-8b/r1/accuracy_results.json | 15 - .../results/system_f/llama3-8b/r1/config.yaml | 1 - .../results/system_f/llama3-8b/r1/point.yaml | 49 -- .../system_f/llama3-8b/r1/result_summary.json | 29 -- .../system_f/llama3-8b/r1/system_desc.json | 60 --- .../llama3-8b/r1024/accuracy_results.json | 15 - .../system_f/llama3-8b/r1024/config.yaml | 1 - .../system_f/llama3-8b/r1024/point.yaml | 50 -- .../llama3-8b/r1024/result_summary.json | 29 -- .../system_f/llama3-8b/r1024/system_desc.json | 60 --- .../llama3-8b/r128/accuracy_results.json | 15 - .../system_f/llama3-8b/r128/config.yaml | 1 - .../system_f/llama3-8b/r128/point.yaml | 49 -- .../llama3-8b/r128/result_summary.json | 29 -- .../system_f/llama3-8b/r128/system_desc.json | 60 --- .../llama3-8b/r16/accuracy_results.json | 15 - .../system_f/llama3-8b/r16/config.yaml | 1 - .../results/system_f/llama3-8b/r16/point.yaml | 49 -- .../llama3-8b/r16/result_summary.json | 29 -- .../system_f/llama3-8b/r16/system_desc.json | 60 --- .../llama3-8b/r2/accuracy_results.json | 15 - .../results/system_f/llama3-8b/r2/config.yaml | 1 - .../results/system_f/llama3-8b/r2/point.yaml | 49 -- .../system_f/llama3-8b/r2/result_summary.json | 29 -- .../system_f/llama3-8b/r2/system_desc.json | 60 --- .../llama3-8b/r256/accuracy_results.json | 15 - .../system_f/llama3-8b/r256/config.yaml | 1 - .../system_f/llama3-8b/r256/point.yaml | 49 -- .../llama3-8b/r256/result_summary.json | 29 -- .../system_f/llama3-8b/r256/system_desc.json | 60 --- .../llama3-8b/r32/accuracy_results.json | 15 - .../system_f/llama3-8b/r32/config.yaml | 1 - .../results/system_f/llama3-8b/r32/point.yaml | 49 -- .../llama3-8b/r32/result_summary.json | 29 -- .../system_f/llama3-8b/r32/system_desc.json | 60 --- .../llama3-8b/r4/accuracy_results.json | 15 - .../results/system_f/llama3-8b/r4/config.yaml | 1 - .../results/system_f/llama3-8b/r4/point.yaml | 49 -- .../system_f/llama3-8b/r4/result_summary.json | 29 -- .../system_f/llama3-8b/r4/system_desc.json | 60 --- .../llama3-8b/r512/accuracy_results.json | 15 - .../system_f/llama3-8b/r512/config.yaml | 1 - .../system_f/llama3-8b/r512/point.yaml | 49 -- .../llama3-8b/r512/result_summary.json | 29 -- .../system_f/llama3-8b/r512/system_desc.json | 60 --- .../llama3-8b/r64/accuracy_results.json | 15 - .../system_f/llama3-8b/r64/config.yaml | 1 - .../results/system_f/llama3-8b/r64/point.yaml | 49 -- .../llama3-8b/r64/result_summary.json | 29 -- .../system_f/llama3-8b/r64/system_desc.json | 60 --- .../llama3-8b/r8/accuracy_results.json | 15 - .../results/system_f/llama3-8b/r8/config.yaml | 1 - .../results/system_f/llama3-8b/r8/point.yaml | 49 -- .../system_f/llama3-8b/r8/result_summary.json | 29 -- .../system_f/llama3-8b/r8/system_desc.json | 60 --- .../sub_f/results/system_f/system_power.json | 68 --- .../submissions/sub_f/src/vllm/README.md | 3 - .../submissions/sub_g/docs/.gitkeep | 0 .../llama3-70b/r1024/accuracy_results.json | 9 - .../system_g/llama3-70b/r1024/config.yaml | 1 - .../system_g/llama3-70b/r1024/point.yaml | 49 -- .../llama3-70b/r1024/result_summary.json | 29 -- .../llama3-70b/r1024/system_desc.json | 61 --- .../llama3-70b/r128/accuracy_results.json | 9 - .../system_g/llama3-70b/r128/config.yaml | 1 - .../system_g/llama3-70b/r128/point.yaml | 49 -- .../llama3-70b/r128/result_summary.json | 29 -- .../system_g/llama3-70b/r128/system_desc.json | 61 --- .../llama3-70b/r1536/accuracy_results.json | 9 - .../system_g/llama3-70b/r1536/config.yaml | 1 - .../system_g/llama3-70b/r1536/point.yaml | 49 -- .../llama3-70b/r1536/result_summary.json | 29 -- .../llama3-70b/r1536/system_desc.json | 61 --- .../llama3-70b/r192/accuracy_results.json | 9 - .../system_g/llama3-70b/r192/config.yaml | 1 - .../system_g/llama3-70b/r192/point.yaml | 49 -- .../llama3-70b/r192/result_summary.json | 29 -- .../system_g/llama3-70b/r192/system_desc.json | 61 --- .../llama3-70b/r2048/accuracy_results.json | 9 - .../system_g/llama3-70b/r2048/config.yaml | 1 - .../system_g/llama3-70b/r2048/point.yaml | 50 -- .../llama3-70b/r2048/result_summary.json | 29 -- .../llama3-70b/r2048/system_desc.json | 61 --- .../llama3-70b/r256/accuracy_results.json | 9 - .../system_g/llama3-70b/r256/config.yaml | 1 - .../system_g/llama3-70b/r256/point.yaml | 49 -- .../llama3-70b/r256/result_summary.json | 29 -- .../system_g/llama3-70b/r256/system_desc.json | 61 --- .../llama3-70b/r384/accuracy_results.json | 9 - .../system_g/llama3-70b/r384/config.yaml | 1 - .../system_g/llama3-70b/r384/point.yaml | 49 -- .../llama3-70b/r384/result_summary.json | 29 -- .../system_g/llama3-70b/r384/system_desc.json | 61 --- .../llama3-70b/r512/accuracy_results.json | 9 - .../system_g/llama3-70b/r512/config.yaml | 1 - .../system_g/llama3-70b/r512/point.yaml | 49 -- .../llama3-70b/r512/result_summary.json | 29 -- .../system_g/llama3-70b/r512/system_desc.json | 61 --- .../llama3-70b/r64/accuracy_results.json | 9 - .../system_g/llama3-70b/r64/config.yaml | 1 - .../system_g/llama3-70b/r64/point.yaml | 49 -- .../llama3-70b/r64/result_summary.json | 29 -- .../system_g/llama3-70b/r64/system_desc.json | 61 --- .../llama3-70b/r768/accuracy_results.json | 9 - .../system_g/llama3-70b/r768/config.yaml | 1 - .../system_g/llama3-70b/r768/point.yaml | 49 -- .../llama3-70b/r768/result_summary.json | 29 -- .../system_g/llama3-70b/r768/system_desc.json | 61 --- .../sub_g/results/system_g/system_power.json | 44 -- .../submissions/sub_g/src/vllm/README.md | 3 - .../submissions/sub_h/docs/.gitkeep | 0 .../llama3-70b/r1024/accuracy_results.json | 9 - .../system_h/llama3-70b/r1024/config.yaml | 1 - .../system_h/llama3-70b/r1024/point.yaml | 49 -- .../llama3-70b/r1024/result_summary.json | 29 -- .../llama3-70b/r1024/system_desc.json | 61 --- .../llama3-70b/r128/accuracy_results.json | 9 - .../system_h/llama3-70b/r128/config.yaml | 1 - .../system_h/llama3-70b/r128/point.yaml | 49 -- .../llama3-70b/r128/result_summary.json | 29 -- .../system_h/llama3-70b/r128/system_desc.json | 61 --- .../llama3-70b/r1536/accuracy_results.json | 9 - .../system_h/llama3-70b/r1536/config.yaml | 1 - .../system_h/llama3-70b/r1536/point.yaml | 49 -- .../llama3-70b/r1536/result_summary.json | 29 -- .../llama3-70b/r1536/system_desc.json | 61 --- .../llama3-70b/r192/accuracy_results.json | 9 - .../system_h/llama3-70b/r192/config.yaml | 1 - .../system_h/llama3-70b/r192/point.yaml | 49 -- .../llama3-70b/r192/result_summary.json | 29 -- .../system_h/llama3-70b/r192/system_desc.json | 61 --- .../llama3-70b/r2048/accuracy_results.json | 9 - .../system_h/llama3-70b/r2048/config.yaml | 1 - .../system_h/llama3-70b/r2048/point.yaml | 50 -- .../llama3-70b/r2048/result_summary.json | 29 -- .../llama3-70b/r2048/system_desc.json | 61 --- .../llama3-70b/r256/accuracy_results.json | 9 - .../system_h/llama3-70b/r256/config.yaml | 1 - .../system_h/llama3-70b/r256/point.yaml | 49 -- .../llama3-70b/r256/result_summary.json | 29 -- .../system_h/llama3-70b/r256/system_desc.json | 61 --- .../llama3-70b/r384/accuracy_results.json | 9 - .../system_h/llama3-70b/r384/config.yaml | 1 - .../system_h/llama3-70b/r384/point.yaml | 49 -- .../llama3-70b/r384/result_summary.json | 29 -- .../system_h/llama3-70b/r384/system_desc.json | 61 --- .../llama3-70b/r512/accuracy_results.json | 9 - .../system_h/llama3-70b/r512/config.yaml | 1 - .../system_h/llama3-70b/r512/point.yaml | 49 -- .../llama3-70b/r512/result_summary.json | 29 -- .../system_h/llama3-70b/r512/system_desc.json | 61 --- .../llama3-70b/r64/accuracy_results.json | 9 - .../system_h/llama3-70b/r64/config.yaml | 1 - .../system_h/llama3-70b/r64/point.yaml | 49 -- .../llama3-70b/r64/result_summary.json | 29 -- .../system_h/llama3-70b/r64/system_desc.json | 61 --- .../llama3-70b/r768/accuracy_results.json | 9 - .../system_h/llama3-70b/r768/config.yaml | 1 - .../system_h/llama3-70b/r768/point.yaml | 49 -- .../llama3-70b/r768/result_summary.json | 29 -- .../system_h/llama3-70b/r768/system_desc.json | 61 --- .../sub_h/results/system_h/system_power.json | 44 -- .../submissions/sub_h/src/sglang/README.md | 3 - .../submissions/sub_i/docs/.gitkeep | 0 .../deepseek-r1/r1/accuracy_results.json | 11 - .../system_i/deepseek-r1/r1/config.yaml | 1 - .../system_i/deepseek-r1/r1/point.yaml | 49 -- .../deepseek-r1/r1/result_summary.json | 29 -- .../system_i/deepseek-r1/r1/system_desc.json | 62 --- .../deepseek-r1/r128/accuracy_results.json | 11 - .../system_i/deepseek-r1/r128/config.yaml | 1 - .../system_i/deepseek-r1/r128/point.yaml | 49 -- .../deepseek-r1/r128/result_summary.json | 29 -- .../deepseek-r1/r128/system_desc.json | 62 --- .../deepseek-r1/r16/accuracy_results.json | 11 - .../system_i/deepseek-r1/r16/config.yaml | 1 - .../system_i/deepseek-r1/r16/point.yaml | 49 -- .../deepseek-r1/r16/result_summary.json | 29 -- .../system_i/deepseek-r1/r16/system_desc.json | 62 --- .../deepseek-r1/r2/accuracy_results.json | 11 - .../system_i/deepseek-r1/r2/config.yaml | 1 - .../system_i/deepseek-r1/r2/point.yaml | 49 -- .../deepseek-r1/r2/result_summary.json | 29 -- .../system_i/deepseek-r1/r2/system_desc.json | 62 --- .../deepseek-r1/r256/accuracy_results.json | 11 - .../system_i/deepseek-r1/r256/config.yaml | 1 - .../system_i/deepseek-r1/r256/point.yaml | 49 -- .../deepseek-r1/r256/result_summary.json | 29 -- .../deepseek-r1/r256/system_desc.json | 62 --- .../deepseek-r1/r32/accuracy_results.json | 11 - .../system_i/deepseek-r1/r32/config.yaml | 1 - .../system_i/deepseek-r1/r32/point.yaml | 49 -- .../deepseek-r1/r32/result_summary.json | 29 -- .../system_i/deepseek-r1/r32/system_desc.json | 62 --- .../deepseek-r1/r4/accuracy_results.json | 11 - .../system_i/deepseek-r1/r4/config.yaml | 1 - .../system_i/deepseek-r1/r4/point.yaml | 49 -- .../deepseek-r1/r4/result_summary.json | 29 -- .../system_i/deepseek-r1/r4/system_desc.json | 62 --- .../deepseek-r1/r512/accuracy_results.json | 11 - .../system_i/deepseek-r1/r512/config.yaml | 1 - .../system_i/deepseek-r1/r512/point.yaml | 50 -- .../deepseek-r1/r512/result_summary.json | 29 -- .../deepseek-r1/r512/system_desc.json | 62 --- .../deepseek-r1/r64/accuracy_results.json | 11 - .../system_i/deepseek-r1/r64/config.yaml | 1 - .../system_i/deepseek-r1/r64/point.yaml | 49 -- .../deepseek-r1/r64/result_summary.json | 29 -- .../system_i/deepseek-r1/r64/system_desc.json | 62 --- .../deepseek-r1/r8/accuracy_results.json | 11 - .../system_i/deepseek-r1/r8/config.yaml | 1 - .../system_i/deepseek-r1/r8/point.yaml | 49 -- .../deepseek-r1/r8/result_summary.json | 29 -- .../system_i/deepseek-r1/r8/system_desc.json | 62 --- .../sub_i/results/system_i/system_power.json | 44 -- .../sub_i/src/tensorrt-llm/README.md | 3 - .../submissions/sub_j/docs/.gitkeep | 0 .../deepseek-r1/r1024/accuracy_results.json | 11 - .../system_j/deepseek-r1/r1024/config.yaml | 1 - .../system_j/deepseek-r1/r1024/point.yaml | 49 -- .../deepseek-r1/r1024/result_summary.json | 29 -- .../deepseek-r1/r1024/system_desc.json | 61 --- .../deepseek-r1/r128/accuracy_results.json | 11 - .../system_j/deepseek-r1/r128/config.yaml | 1 - .../system_j/deepseek-r1/r128/point.yaml | 49 -- .../deepseek-r1/r128/result_summary.json | 29 -- .../deepseek-r1/r128/system_desc.json | 61 --- .../deepseek-r1/r16384/accuracy_results.json | 11 - .../system_j/deepseek-r1/r16384/config.yaml | 1 - .../system_j/deepseek-r1/r16384/point.yaml | 50 -- .../deepseek-r1/r16384/result_summary.json | 29 -- .../deepseek-r1/r16384/system_desc.json | 61 --- .../deepseek-r1/r2048/accuracy_results.json | 11 - .../system_j/deepseek-r1/r2048/config.yaml | 1 - .../system_j/deepseek-r1/r2048/point.yaml | 49 -- .../deepseek-r1/r2048/result_summary.json | 29 -- .../deepseek-r1/r2048/system_desc.json | 61 --- .../deepseek-r1/r256/accuracy_results.json | 11 - .../system_j/deepseek-r1/r256/config.yaml | 1 - .../system_j/deepseek-r1/r256/point.yaml | 49 -- .../deepseek-r1/r256/result_summary.json | 29 -- .../deepseek-r1/r256/system_desc.json | 61 --- .../deepseek-r1/r32/accuracy_results.json | 11 - .../system_j/deepseek-r1/r32/config.yaml | 1 - .../system_j/deepseek-r1/r32/point.yaml | 49 -- .../deepseek-r1/r32/result_summary.json | 29 -- .../system_j/deepseek-r1/r32/system_desc.json | 61 --- .../deepseek-r1/r4096/accuracy_results.json | 11 - .../system_j/deepseek-r1/r4096/config.yaml | 1 - .../system_j/deepseek-r1/r4096/point.yaml | 49 -- .../deepseek-r1/r4096/result_summary.json | 29 -- .../deepseek-r1/r4096/system_desc.json | 61 --- .../deepseek-r1/r512/accuracy_results.json | 11 - .../system_j/deepseek-r1/r512/config.yaml | 1 - .../system_j/deepseek-r1/r512/point.yaml | 49 -- .../deepseek-r1/r512/result_summary.json | 29 -- .../deepseek-r1/r512/system_desc.json | 61 --- .../deepseek-r1/r64/accuracy_results.json | 11 - .../system_j/deepseek-r1/r64/config.yaml | 1 - .../system_j/deepseek-r1/r64/point.yaml | 49 -- .../deepseek-r1/r64/result_summary.json | 29 -- .../system_j/deepseek-r1/r64/system_desc.json | 61 --- .../deepseek-r1/r8192/accuracy_results.json | 11 - .../system_j/deepseek-r1/r8192/config.yaml | 1 - .../system_j/deepseek-r1/r8192/point.yaml | 49 -- .../deepseek-r1/r8192/result_summary.json | 29 -- .../deepseek-r1/r8192/system_desc.json | 61 --- .../sub_j/results/system_j/system_power.json | 44 -- .../sub_j/src/tensorrt-llm/README.md | 3 - .../valid_standardized/docs/.gitkeep | 0 .../valid_standardized/docs/calibration.adoc | 5 - .../llama3_1-8b/r1000/accuracy_results.json | 13 - .../system_k/llama3_1-8b/r1000/config.yaml | 1 - .../system_k/llama3_1-8b/r1000/point.yaml | 50 -- .../llama3_1-8b/r1000/result_summary.json | 29 -- .../llama3_1-8b/r1000/system_desc.json | 63 --- .../llama3_1-8b/r16/accuracy_results.json | 13 - .../system_k/llama3_1-8b/r16/config.yaml | 1 - .../system_k/llama3_1-8b/r16/point.yaml | 49 -- .../llama3_1-8b/r16/result_summary.json | 29 -- .../system_k/llama3_1-8b/r16/system_desc.json | 63 --- .../llama3_1-8b/r20/accuracy_results.json | 13 - .../system_k/llama3_1-8b/r20/config.yaml | 1 - .../system_k/llama3_1-8b/r20/point.yaml | 49 -- .../llama3_1-8b/r20/result_summary.json | 29 -- .../system_k/llama3_1-8b/r20/system_desc.json | 63 --- .../llama3_1-8b/r256/accuracy_results.json | 13 - .../system_k/llama3_1-8b/r256/config.yaml | 1 - .../system_k/llama3_1-8b/r256/point.yaml | 49 -- .../llama3_1-8b/r256/result_summary.json | 29 -- .../llama3_1-8b/r256/system_desc.json | 63 --- .../llama3_1-8b/r512/accuracy_results.json | 13 - .../system_k/llama3_1-8b/r512/config.yaml | 1 - .../system_k/llama3_1-8b/r512/point.yaml | 49 -- .../llama3_1-8b/r512/result_summary.json | 29 -- .../llama3_1-8b/r512/system_desc.json | 63 --- .../llama3_1-8b/r768/accuracy_results.json | 13 - .../system_k/llama3_1-8b/r768/config.yaml | 1 - .../system_k/llama3_1-8b/r768/point.yaml | 49 -- .../llama3_1-8b/r768/result_summary.json | 29 -- .../llama3_1-8b/r768/system_desc.json | 63 --- .../llama3_1-8b/r88/accuracy_results.json | 13 - .../system_k/llama3_1-8b/r88/config.yaml | 1 - .../system_k/llama3_1-8b/r88/point.yaml | 49 -- .../llama3_1-8b/r88/result_summary.json | 29 -- .../system_k/llama3_1-8b/r88/system_desc.json | 63 --- .../results/system_k/system_power.json | 44 -- .../src/llama3_1-8b/.gitkeep | 0 .../src/llama3_1-8b/README.md | 3 - .../valid_standardized/src/vllm/README.md | 3 - tests/unit/validation/conftest.py | 67 +++ tests/unit/validation/test_engine.py | 111 ++-- .../test_final_review_regressions.py | 11 +- .../unit/validation/test_policy_contracts.py | 6 +- 561 files changed, 4948 insertions(+), 15940 deletions(-) create mode 100644 tests/fixtures/validation/cases/invalid_submission.yaml create mode 100644 tests/fixtures/validation/cases/sub_a.yaml create mode 100644 tests/fixtures/validation/cases/sub_b.yaml create mode 100644 tests/fixtures/validation/cases/sub_c.yaml create mode 100644 tests/fixtures/validation/cases/sub_d.yaml create mode 100644 tests/fixtures/validation/cases/sub_e.yaml create mode 100644 tests/fixtures/validation/cases/sub_f.yaml create mode 100644 tests/fixtures/validation/cases/sub_g.yaml create mode 100644 tests/fixtures/validation/cases/sub_h.yaml create mode 100644 tests/fixtures/validation/cases/sub_i.yaml create mode 100644 tests/fixtures/validation/cases/sub_j.yaml create mode 100644 tests/fixtures/validation/cases/valid_standardized.yaml delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/docs/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/config.yaml delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/point.yaml delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/config.yaml delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/point.yaml delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/config.yaml delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/point.yaml delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/results/system_l/system_power.json delete mode 100644 tests/fixtures/validation/submissions/invalid_submission/src/custom/README.md delete mode 100644 tests/fixtures/validation/submissions/sub_a/docs/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/results/system_a/system_power.json delete mode 100644 tests/fixtures/validation/submissions/sub_a/src/vllm/README.md delete mode 100644 tests/fixtures/validation/submissions/sub_b/docs/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/results/system_b/system_power.json delete mode 100644 tests/fixtures/validation/submissions/sub_b/src/vllm/README.md delete mode 100644 tests/fixtures/validation/submissions/sub_c/docs/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/results/system_c/system_power.json delete mode 100644 tests/fixtures/validation/submissions/sub_c/src/vllm/README.md delete mode 100644 tests/fixtures/validation/submissions/sub_d/docs/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/results/system_d/system_power.json delete mode 100644 tests/fixtures/validation/submissions/sub_d/src/vllm/README.md delete mode 100644 tests/fixtures/validation/submissions/sub_e/docs/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/results/system_e/system_power.json delete mode 100644 tests/fixtures/validation/submissions/sub_e/src/vllm/README.md delete mode 100644 tests/fixtures/validation/submissions/sub_f/docs/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/results/system_f/system_power.json delete mode 100644 tests/fixtures/validation/submissions/sub_f/src/vllm/README.md delete mode 100644 tests/fixtures/validation/submissions/sub_g/docs/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/results/system_g/system_power.json delete mode 100644 tests/fixtures/validation/submissions/sub_g/src/vllm/README.md delete mode 100644 tests/fixtures/validation/submissions/sub_h/docs/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/results/system_h/system_power.json delete mode 100644 tests/fixtures/validation/submissions/sub_h/src/sglang/README.md delete mode 100644 tests/fixtures/validation/submissions/sub_i/docs/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/results/system_i/system_power.json delete mode 100644 tests/fixtures/validation/submissions/sub_i/src/tensorrt-llm/README.md delete mode 100644 tests/fixtures/validation/submissions/sub_j/docs/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/config.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/point.yaml delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/results/system_j/system_power.json delete mode 100644 tests/fixtures/validation/submissions/sub_j/src/tensorrt-llm/README.md delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/docs/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/docs/calibration.adoc delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/config.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/point.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/config.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/point.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/config.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/point.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/config.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/point.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/config.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/point.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/config.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/point.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/accuracy_results.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/config.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/point.yaml delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/result_summary.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/system_desc.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/results/system_k/system_power.json delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/.gitkeep delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/README.md delete mode 100644 tests/fixtures/validation/submissions/valid_standardized/src/vllm/README.md create mode 100644 tests/unit/validation/conftest.py diff --git a/tests/fixtures/validation/README.md b/tests/fixtures/validation/README.md index 3ef929d70..4b01ab77b 100644 --- a/tests/fixtures/validation/README.md +++ b/tests/fixtures/validation/README.md @@ -1,25 +1,31 @@ # Validation fixtures -The submission trees are synthetic test cases, not historical submissions or +The submission cases are synthetic test inputs, not historical submissions or measured results. Their accuracy scores, performance summaries, steady-state -windows, system configurations, and power disclosures are constructed to exercise -validation behavior. They must not be used to compare hardware or serving software. +windows, system configurations, and power disclosures exercise validation behavior. +They must not be used to compare hardware or serving software. -Each invented system has a neutral identifier (`system_a` through `system_l`). -Real model, dataset, accelerator, and processor names remain where they exercise -published policy keys and hardware power lookups; these names do not establish -that the configurations were run on those products. +`cases/` contains one readable YAML file per submission. Each file has: -`valid_standardized` supplies a complete standardized submission layout. -`invalid_submission` exercises invalid declarations and missing requirements. -`sub_a` through `sub_j` exercise different curve, model, configuration, and power -paths. These are checker inputs: the directory names do not guarantee that every -case passes all rules in the current cohort. +- `defaults`: common artifact fields shared by its measurement points. +- `files`: submission-level artifacts and small source/documentation files. +- `points`: point paths and artifact fields that differ from the defaults. -The corpus was adapted from the submission checker test corpus. Its fixture -regeneration tool explicitly describes the data as synthetic and constructs -accuracy values, steady-state windows, and power disclosures. The copies here -are maintained as native validation fixtures and require no external checker. +The pytest fixtures in `tests/unit/validation/conftest.py` write complete JSON and +YAML artifacts into a temporary directory. Nested dictionaries merge recursively; +scalar values and lists replace their defaults. An empty override uses all defaults. +An omitted artifact is absent; `null` is a value, not a deletion instruction. +Missing fields must be omitted from defaults and supplied only by points that need +them. Tests can copy and mutate generated trees without changing the case files. + +`valid_standardized` supplies a complete standardized layout. `invalid_submission` +exercises invalid declarations and missing requirements. `sub_a` through `sub_j` +exercise different curves, models, configurations, and power paths. Their names do +not guarantee that every case passes all rules in the current cohort. + +Invented systems use neutral identifiers (`system_a` through `system_l`). Real +model, dataset, accelerator, and processor names exercise published policy keys +and hardware power lookups; they do not establish that those configurations ran. `policy-fingerprints.json` pins the catalog and rule definitions independently of -these submission trees, so policy edits require an explicit expected-value update. +these cases, so policy edits require an explicit expected-value update. diff --git a/tests/fixtures/validation/cases/invalid_submission.yaml b/tests/fixtures/validation/cases/invalid_submission.yaml new file mode 100644 index 000000000..4cec01356 --- /dev/null +++ b/tests/fixtures/validation/cases/invalid_submission.yaml @@ -0,0 +1,250 @@ +defaults: + accuracy_results.json: + cnn_dailymail::llama3_8b: + dataset_name: cnn_dailymail::llama3_8b + num_samples: 500 + extractor: RougeExtractor + score: + rouge1: "30.00" + rouge2: "12.00" + rougeL: "20.00" + n_repeats: 1 + config.yaml: {} + point.yaml: + dataset: llm-perf-dataset-v1 + dataset_link: https://example.com/dataset + dataset_name: CNN/DailyMail + dataset_type: Performance + division: Standardized + link_to_model: https://example.com/model + link_to_model_transformation: https://example.com/quantization + max_supported_concurrency: 88 + model_name: Llama-3_1-8B-Instruct + model_notes: "" + model_precision: FP16 + runtime_settings: + load_pattern: concurrency + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true + seed_set: A + shared_docs: docs + shared_src: src + steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 + target_cohort: 2026-10-C1 + warmup: + data_source: llm-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + result_summary.json: + n_samples_failed: 0 + ttft: + percentiles: + "50": 45200000.0 + "95": 89100000.0 + "90": 80320000.0 + output_sequence_lengths: + percentiles: {} + tpot: + percentiles: + "50": 4000000.0 + "90": 5500000.0 + "95": 6000000.0 + "99": 6500000.0 + tps_per_user: 181.8182 + system_desc.json: + submitter_org_names: Invalid Corp + submitter_contact: contact@example.com + system_name: system_l + system_category: datacenter + max_supported_concurrency: 88 + serving_framework: custom 0.0.1 + node_types: + - system_node_ensemble_id: 0 + number_of_nodes: 1 + host_processor_model_name: Generic CPU + host_processors_per_node: 1 + host_processor_core_count: 8 + host_memory_capacity: 64 GB + host_networking: Ethernet 1Gb + host_storage_capacity: 1 TB + host_storage_type: HDD + operating_system: Ubuntu 20.04 + host_memory_configuration: 8x 64GB DDR5 + host_network_card_count: 1x NIC + driver: "1.0" + filesystem: ext4 + accelerator_info: + - accelerator_model_name: Generic GPU + accelerators_per_node: 1 + accelerator_memory_capacity: 16 GB + accelerator_memory_type: HBM3e + division: Standardized + model_precision: FP16 + link_to_model: https://example.com/model + dataset_id: cnn_dailymail + dataset_name: CNN/DailyMail + input_token_average: 870.0 + output_token_average: 128.0 + dataset_type: performance + dataset_link: https://example.com/dataset + measured_accuracy_score: "38.7" + system_size: 1 node + model_id: Llama-3_1-8B-Instruct + system_node_ensemble_count: 1 + system_node_ensemble_total: 1 + publication_status: Available + node_config: 8x H100 test node + config_summary: + disaggregated: null + expert_parallel: 1 + tensor_parallel: 1 + pipeline_parallel: 1 + data_parallel: 1 + batch: null + cooling: Air-cooled +files: + docs/.gitkeep: "" + results/system_l/system_power.json: + system_desc_id: system_l + cooling: air + node_sets: + - node_set_id: 0 + system_node_ensemble_id: 0 + nodes_provisioned: 1 + power_method: component_sum + components: + cpu: + model: Generic CPU + count_per_node: 1 + tdp_per_unit: + value_w: 350 + source_type: vendor_spec + source: https://example.com/cpu-spec + accelerator: + model: Generic GPU + count_per_node: 1 + tdp_per_unit: + value_w: 700 + source_type: vendor_spec + source: https://example.com/accelerator-spec + scale_up_network: + method: declared_tdp + switch_count: 1 + tdp_per_switch: + value_w: 3500 + source_type: vendor_spec + source: https://example.com/switch-spec + scale_out: + present: false + src/custom/README.md: "# custom + + + Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce + a point here. + + " +points: + results/system_l/Llama-3_1-8B-Instruct/r16: + accuracy_results.json: {} + config.yaml: + concurrency: 16 + point.yaml: + concurrency: 16 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 16 + requests_completed: 160 + requests_issued: 160 + result_summary.json: + n_samples_issued: 3000 + n_samples_completed: 3000 + duration_ns: 600000000000 + ttft: + total: 135600000000.0 + output_sequence_lengths: + total: 480000.0 + tpot: + total: 12147000000.000002 + system_tps: 800.0 + system_desc.json: + tps_utilization: 0.32 + results/system_l/Llama-3_1-8B-Instruct/r38: + accuracy_results.json: {} + config.yaml: + concurrency: 38 + point.yaml: + concurrency: 38 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 38 + requests_completed: 380 + requests_issued: 380 + result_summary.json: + n_samples_issued: 8000 + n_samples_completed: 8000 + duration_ns: 1200000000000 + ttft: + total: 361600000000.0 + output_sequence_lengths: + total: 1800000.0 + tpot: + total: 32392000000.000004 + system_tps: 1500.0 + system_desc.json: + tps_utilization: 0.6 + results/system_l/Llama-3_1-8B-Instruct/r88: + accuracy_results.json: {} + config.yaml: + concurrency: 88 + point.yaml: + concurrency: 88 + offline: elected + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 88 + requests_completed: 880 + requests_issued: 880 + result_summary.json: + n_samples_issued: 5000 + n_samples_completed: 5000 + duration_ns: 1200000000000 + ttft: + total: 226000000000.0 + output_sequence_lengths: + total: 3000000.0 + tpot: + total: 20245000000.000004 + system_tps: 2500.0 + system_desc.json: + tps_utilization: 1.0 diff --git a/tests/fixtures/validation/cases/sub_a.yaml b/tests/fixtures/validation/cases/sub_a.yaml new file mode 100644 index 000000000..5a5a4828c --- /dev/null +++ b/tests/fixtures/validation/cases/sub_a.yaml @@ -0,0 +1,358 @@ +defaults: + accuracy_results.json: + cnn_dailymail_validation: + dataset_name: cnn_dailymail_validation + num_samples: 4395 + score: + exact_match: 83.13 + extractor: RougeExtractor + n_repeats: 1 + config.yaml: {} + point.yaml: + dataset: mlperf-perf-dataset-v1 + dataset_link: https://example.com/dataset + dataset_name: CNN/DailyMail + dataset_type: Performance + division: Standardized + link_to_model: https://example.com/model + link_to_model_transformation: https://example.com/quantization + max_supported_concurrency: 2048 + model_name: gpt-oss-120b + model_notes: "" + model_precision: FP16 + runtime_settings: + load_pattern: concurrency + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true + seed_set: A + shared_docs: docs + shared_src: src + steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 + target_cohort: 2026-10-C1 + warmup: + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + result_summary.json: + n_samples_issued: 6396 + n_samples_completed: 6396 + n_samples_failed: 0 + output_sequence_lengths: + percentiles: {} + tpot: + total: 25897404000.000004 + percentiles: + "50": 4000000.0 + "90": 5500000.0 + "95": 6000000.0 + "99": 6500000.0 + tps_per_user: 181.8182 + system_desc.json: + submitter_org_names: Submitter-A + submitter_contact: contact@example.com + system_name: system_a + system_category: datacenter + max_supported_concurrency: 2048 + serving_framework: vLLM + node_types: + - system_node_ensemble_id: 0 + number_of_nodes: 1 + host_processor_model_name: AMD EPYC 9654 + host_processors_per_node: 2 + host_processor_core_count: 96 + host_memory_capacity: 768 GB + host_networking: InfiniBand HDR + host_storage_capacity: 30 TB + host_storage_type: NVMe SSD + operating_system: Ubuntu 22.04 + host_memory_configuration: 8x 64GB DDR5 + host_network_card_count: 1x NIC + driver: "1.0" + filesystem: ext4 + accelerator_info: + - accelerator_model_name: AMD Instinct MI355X + accelerators_per_node: 8 + accelerator_memory_capacity: 288 GB + accelerator_memory_type: HBM3e + division: Standardized + model_precision: FP16 + link_to_model: https://example.com/model + dataset_id: cnn_dailymail + dataset_name: CNN/DailyMail + input_token_average: 870.0 + output_token_average: 128.0 + dataset_type: performance + dataset_link: https://example.com/dataset + measured_accuracy_score: "38.7" + system_size: 1 node + model_id: gpt-oss-120b + system_node_ensemble_count: 1 + system_node_ensemble_total: 1 + publication_status: Available + node_config: 8x H100 test node + config_summary: + disaggregated: null + expert_parallel: 1 + tensor_parallel: 8 + pipeline_parallel: 1 + data_parallel: 1 + batch: null + cooling: Air-cooled +files: + docs/.gitkeep: "" + results/system_a/system_power.json: + system_desc_id: system_a + cooling: air + node_sets: + - node_set_id: 0 + system_node_ensemble_id: 0 + nodes_provisioned: 1 + power_method: component_sum + components: + cpu: + model: AMD EPYC 9654 + count_per_node: 2 + tdp_per_unit: + value_w: 350 + source_type: vendor_spec + source: https://example.com/cpu-spec + accelerator: + model: AMD Instinct MI355X + count_per_node: 8 + tdp_per_unit: + value_w: 700 + source_type: vendor_spec + source: https://example.com/accelerator-spec + scale_up_network: + method: declared_tdp + switch_count: 1 + tdp_per_switch: + value_w: 3500 + source_type: vendor_spec + source: https://example.com/switch-spec + scale_out: + present: false + src/vllm/README.md: "# vllm + + + Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce + a point here. + + " +points: + results/system_a/gpt-oss-120b/r1024: + accuracy_results.json: {} + config.yaml: + concurrency: 1024 + point.yaml: + concurrency: 1024 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 1024 + requests_completed: 10240 + requests_issued: 10240 + result_summary.json: + duration_ns: 2302757311892.0 + ttft: + total: 551183759784528.0 + percentiles: + "50": 86176322668.0 + "95": 100733262106.25 + "90": 97821874218.6 + output_sequence_lengths: + total: 22417180.0 + system_tps: 9734.9295 + system_desc.json: + tps_utilization: 0.997637 + results/system_a/gpt-oss-120b/r128: + accuracy_results.json: {} + config.yaml: + concurrency: 128 + point.yaml: + concurrency: 128 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 128 + requests_completed: 1280 + requests_issued: 1280 + result_summary.json: + duration_ns: 3940004920032.0 + ttft: + total: 1084206049836.0 + percentiles: + "50": 169513141.0 + "95": 261888051.50000003 + "90": 243413069.40000004 + output_sequence_lengths: + total: 22407098.0 + system_tps: 5687.0736 + system_desc.json: + tps_utilization: 0.582812 + results/system_a/gpt-oss-120b/r16: + accuracy_results.json: {} + config.yaml: + concurrency: 16 + point.yaml: + concurrency: 16 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 16 + requests_completed: 160 + requests_issued: 160 + result_summary.json: + duration_ns: 11856610466381.0 + ttft: + total: 599489491146.0 + percentiles: + "50": 93728813.5 + "95": 145818384.25 + "90": 135400470.1 + output_sequence_lengths: + total: 22387661.0 + system_tps: 1888.2008 + system_desc.json: + tps_utilization: 0.193503 + results/system_a/gpt-oss-120b/r2048: + accuracy_results.json: {} + config.yaml: + concurrency: 2048 + point.yaml: + concurrency: 2048 + offline: elected + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 2048 + requests_completed: 20480 + requests_issued: 20480 + result_summary.json: + duration_ns: 2294246323608.9995 + ttft: + total: 2845856057894070.0 + percentiles: + "50": 444943098482.5 + "95": 462411343300.0 + "90": 458917694336.5 + output_sequence_lengths: + total: 22387231.0 + system_tps: 9757.9893 + system_desc.json: + tps_utilization: 1.0 + results/system_a/gpt-oss-120b/r4: + accuracy_results.json: {} + config.yaml: + concurrency: 4 + point.yaml: + concurrency: 4 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 4 + requests_completed: 40 + requests_issued: 40 + result_summary.json: + duration_ns: 28913949735508.0 + ttft: + total: 521833969800.0 + percentiles: + "50": 81587550.0 + "95": 183622220.0 + "90": 163215286.0 + output_sequence_lengths: + total: 22427551.0 + system_tps: 775.6654 + system_desc.json: + tps_utilization: 0.07949 + results/system_a/gpt-oss-120b/r512: + accuracy_results.json: {} + config.yaml: + concurrency: 512 + point.yaml: + concurrency: 512 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 512 + requests_completed: 5120 + requests_issued: 5120 + result_summary.json: + duration_ns: 2427693763211.0 + ttft: + total: 2446935638394.0 + percentiles: + "50": 382572801.5 + "95": 10778207187.0 + "90": 8699080309.900002 + output_sequence_lengths: + total: 22315593.0 + system_tps: 9192.0955 + system_desc.json: + tps_utilization: 0.942007 + results/system_a/gpt-oss-120b/r64: + accuracy_results.json: {} + config.yaml: + concurrency: 64 + point.yaml: + concurrency: 64 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 64 + requests_completed: 640 + requests_issued: 640 + result_summary.json: + duration_ns: 5163749763587.001 + ttft: + total: 814271667534.0 + percentiles: + "50": 127309516.5 + "95": 193387620.0 + "90": 180171999.3 + output_sequence_lengths: + total: 22246057.0 + system_tps: 4308.1207 + system_desc.json: + tps_utilization: 0.441497 diff --git a/tests/fixtures/validation/cases/sub_b.yaml b/tests/fixtures/validation/cases/sub_b.yaml new file mode 100644 index 000000000..9750c2e0a --- /dev/null +++ b/tests/fixtures/validation/cases/sub_b.yaml @@ -0,0 +1,358 @@ +defaults: + accuracy_results.json: + cnn_dailymail_validation: + dataset_name: cnn_dailymail_validation + num_samples: 4395 + score: + exact_match: 83.13 + extractor: RougeExtractor + n_repeats: 1 + config.yaml: {} + point.yaml: + dataset: mlperf-perf-dataset-v1 + dataset_link: https://example.com/dataset + dataset_name: CNN/DailyMail + dataset_type: Performance + division: Standardized + link_to_model: https://example.com/model + link_to_model_transformation: https://example.com/quantization + max_supported_concurrency: 2048 + model_name: gpt-oss-120b + model_notes: "" + model_precision: FP16 + runtime_settings: + load_pattern: concurrency + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true + seed_set: A + shared_docs: docs + shared_src: src + steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 + target_cohort: 2026-10-C1 + warmup: + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + result_summary.json: + n_samples_issued: 6396 + n_samples_completed: 6396 + n_samples_failed: 0 + output_sequence_lengths: + percentiles: {} + tpot: + total: 25897404000.000004 + percentiles: + "50": 4000000.0 + "90": 5500000.0 + "95": 6000000.0 + "99": 6500000.0 + tps_per_user: 181.8182 + system_desc.json: + submitter_org_names: Submitter-B + submitter_contact: contact@example.com + system_name: system_b + system_category: datacenter + max_supported_concurrency: 2048 + serving_framework: vLLM + node_types: + - system_node_ensemble_id: 0 + number_of_nodes: 1 + host_processor_model_name: AMD EPYC 9654 + host_processors_per_node: 2 + host_processor_core_count: 96 + host_memory_capacity: 768 GB + host_networking: InfiniBand HDR + host_storage_capacity: 30 TB + host_storage_type: NVMe SSD + operating_system: Ubuntu 22.04 + host_memory_configuration: 8x 64GB DDR5 + host_network_card_count: 1x NIC + driver: "1.0" + filesystem: ext4 + accelerator_info: + - accelerator_model_name: AMD Instinct MI355X + accelerators_per_node: 16 + accelerator_memory_capacity: 288 GB + accelerator_memory_type: HBM3e + division: Standardized + model_precision: FP16 + link_to_model: https://example.com/model + dataset_id: cnn_dailymail + dataset_name: CNN/DailyMail + input_token_average: 870.0 + output_token_average: 128.0 + dataset_type: performance + dataset_link: https://example.com/dataset + measured_accuracy_score: "38.7" + system_size: 1 node + model_id: gpt-oss-120b + system_node_ensemble_count: 1 + system_node_ensemble_total: 1 + publication_status: Available + node_config: 8x H100 test node + config_summary: + disaggregated: null + expert_parallel: 1 + tensor_parallel: 16 + pipeline_parallel: 1 + data_parallel: 1 + batch: null + cooling: Air-cooled +files: + docs/.gitkeep: "" + results/system_b/system_power.json: + system_desc_id: system_b + cooling: air + node_sets: + - node_set_id: 0 + system_node_ensemble_id: 0 + nodes_provisioned: 1 + power_method: component_sum + components: + cpu: + model: AMD EPYC 9654 + count_per_node: 2 + tdp_per_unit: + value_w: 350 + source_type: vendor_spec + source: https://example.com/cpu-spec + accelerator: + model: AMD Instinct MI355X + count_per_node: 16 + tdp_per_unit: + value_w: 700 + source_type: vendor_spec + source: https://example.com/accelerator-spec + scale_up_network: + method: declared_tdp + switch_count: 1 + tdp_per_switch: + value_w: 3500 + source_type: vendor_spec + source: https://example.com/switch-spec + scale_out: + present: false + src/vllm/README.md: "# vllm + + + Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce + a point here. + + " +points: + results/system_b/gpt-oss-120b/r1024: + accuracy_results.json: {} + config.yaml: + concurrency: 1024 + point.yaml: + concurrency: 1024 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 1024 + requests_completed: 10240 + requests_issued: 10240 + result_summary.json: + duration_ns: 2555878737455.0 + ttft: + total: 607712738731644.0 + percentiles: + "50": 95014499489.0 + "95": 113055413992.5 + "90": 109447231091.8 + output_sequence_lengths: + total: 22287872.0 + system_tps: 8720.2384 + system_desc.json: + tps_utilization: 1.0 + results/system_b/gpt-oss-120b/r128: + accuracy_results.json: {} + config.yaml: + concurrency: 128 + point.yaml: + concurrency: 128 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 128 + requests_completed: 1280 + requests_issued: 1280 + result_summary.json: + duration_ns: 4564665788130.0 + ttft: + total: 1276608945222.0 + percentiles: + "50": 199594894.5 + "95": 306682164.0 + "90": 285264710.1 + output_sequence_lengths: + total: 22291611.0 + system_tps: 4883.5144 + system_desc.json: + tps_utilization: 0.560021 + results/system_b/gpt-oss-120b/r16: + accuracy_results.json: {} + config.yaml: + concurrency: 16 + point.yaml: + concurrency: 16 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 16 + requests_completed: 160 + requests_issued: 160 + result_summary.json: + duration_ns: 13160683481153.0 + ttft: + total: 707502001908.0 + percentiles: + "50": 110616323.0 + "95": 220638667.75 + "90": 198634198.8 + output_sequence_lengths: + total: 22403343.0 + system_tps: 1702.2933 + system_desc.json: + tps_utilization: 0.195212 + results/system_b/gpt-oss-120b/r2048: + accuracy_results.json: {} + config.yaml: + concurrency: 2048 + point.yaml: + concurrency: 2048 + offline: elected + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 2048 + requests_completed: 20480 + requests_issued: 20480 + result_summary.json: + duration_ns: 2587004792635.0 + ttft: + total: 3198282303472386.0 + percentiles: + "50": 500044137503.5 + "95": 545254932484.0 + "90": 536212773487.9 + output_sequence_lengths: + total: 22359894.0 + system_tps: 8643.1591 + system_desc.json: + tps_utilization: 0.991161 + results/system_b/gpt-oss-120b/r4: + accuracy_results.json: {} + config.yaml: + concurrency: 4 + point.yaml: + concurrency: 4 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 4 + requests_completed: 40 + requests_issued: 40 + result_summary.json: + duration_ns: 30408127168980.996 + ttft: + total: 562383145116.0 + percentiles: + "50": 87927321.0 + "95": 197007595.0 + "90": 175191540.2 + output_sequence_lengths: + total: 22330923.0 + system_tps: 734.3735 + system_desc.json: + tps_utilization: 0.084215 + results/system_b/gpt-oss-120b/r512: + accuracy_results.json: {} + config.yaml: + concurrency: 512 + point.yaml: + concurrency: 512 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 512 + requests_completed: 5120 + requests_issued: 5120 + result_summary.json: + duration_ns: 2755067877868.0 + ttft: + total: 2813058879462.0 + percentiles: + "50": 439815334.5 + "95": 11423621862.5 + "90": 9226860556.9 + output_sequence_lengths: + total: 22400639.0 + system_tps: 8130.7031 + system_desc.json: + tps_utilization: 0.932395 + results/system_b/gpt-oss-120b/r64: + accuracy_results.json: {} + config.yaml: + concurrency: 64 + point.yaml: + concurrency: 64 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 64 + requests_completed: 640 + requests_issued: 640 + result_summary.json: + duration_ns: 6369959177476.999 + ttft: + total: 994203277548.0 + percentiles: + "50": 155441413.0 + "95": 237329605.75 + "90": 220951967.2 + output_sequence_lengths: + total: 22395866.0 + system_tps: 3515.8571 + system_desc.json: + tps_utilization: 0.403184 diff --git a/tests/fixtures/validation/cases/sub_c.yaml b/tests/fixtures/validation/cases/sub_c.yaml new file mode 100644 index 000000000..269d02199 --- /dev/null +++ b/tests/fixtures/validation/cases/sub_c.yaml @@ -0,0 +1,357 @@ +defaults: + accuracy_results.json: + humaneval: + dataset_name: humaneval + num_samples: 500 + score: 0.45 + extractor: ExactMatchExtractor + n_repeats: 1 + config.yaml: {} + point.yaml: + dataset: mlperf-perf-dataset-v1 + dataset_link: https://example.com/dataset + dataset_name: CNN/DailyMail + dataset_type: Performance + division: Standardized + link_to_model: https://example.com/model + link_to_model_transformation: https://example.com/quantization + max_supported_concurrency: 512 + model_name: qwen3-coder-480b + model_notes: "" + model_precision: FP16 + runtime_settings: + load_pattern: concurrency + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true + seed_set: A + shared_docs: docs + shared_src: src + steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 + target_cohort: 2026-10-C1 + warmup: + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + result_summary.json: + n_samples_issued: 24576 + n_samples_completed: 24576 + n_samples_failed: 0 + output_sequence_lengths: + percentiles: {} + tpot: + total: 99508224000.00002 + percentiles: + "50": 4000000.0 + "90": 5500000.0 + "95": 6000000.0 + "99": 6500000.0 + tps_per_user: 181.8182 + system_desc.json: + submitter_org_names: Submitter-C + submitter_contact: contact@example.com + system_name: system_c + system_category: datacenter + max_supported_concurrency: 512 + serving_framework: vLLM (TPU) + node_types: + - system_node_ensemble_id: 0 + number_of_nodes: 1 + host_processor_model_name: Custom TPU Host CPU + host_processors_per_node: 1 + host_processor_core_count: 24 + host_memory_capacity: 192 GB + host_networking: Ethernet 100Gb + host_storage_capacity: 10 TB + host_storage_type: NVMe SSD + operating_system: Debian 11 + host_memory_configuration: 8x 64GB DDR5 + host_network_card_count: 1x NIC + driver: "1.0" + filesystem: ext4 + accelerator_info: + - accelerator_model_name: Google TPU v5p + accelerators_per_node: 4 + accelerator_memory_capacity: 96 GB HBM2e + accelerator_memory_type: HBM3e + division: Standardized + model_precision: FP16 + link_to_model: https://example.com/model + dataset_id: cnn_dailymail + dataset_name: CNN/DailyMail + input_token_average: 870.0 + output_token_average: 128.0 + dataset_type: performance + dataset_link: https://example.com/dataset + measured_accuracy_score: "38.7" + system_size: 1 node + model_id: qwen3-coder-480b + system_node_ensemble_count: 1 + system_node_ensemble_total: 1 + publication_status: Available + node_config: 8x H100 test node + config_summary: + disaggregated: null + expert_parallel: 1 + tensor_parallel: 4 + pipeline_parallel: 1 + data_parallel: 1 + batch: null + cooling: Liquid-cooled +files: + docs/.gitkeep: "" + results/system_c/system_power.json: + system_desc_id: system_c + cooling: liquid + node_sets: + - node_set_id: 0 + system_node_ensemble_id: 0 + nodes_provisioned: 1 + power_method: component_sum + components: + cpu: + model: Custom TPU Host CPU + count_per_node: 1 + tdp_per_unit: + value_w: 350 + source_type: vendor_spec + source: https://example.com/cpu-spec + accelerator: + model: Google TPU v5p + count_per_node: 4 + tdp_per_unit: + value_w: 700 + source_type: vendor_spec + source: https://example.com/accelerator-spec + scale_up_network: + method: declared_tdp + switch_count: 1 + tdp_per_switch: + value_w: 3500 + source_type: vendor_spec + source: https://example.com/switch-spec + scale_out: + present: false + src/vllm/README.md: "# vllm + + + Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce + a point here. + + " +points: + results/system_c/qwen3-coder-480b/r128: + accuracy_results.json: {} + config.yaml: + concurrency: 128 + point.yaml: + concurrency: 128 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 128 + requests_completed: 1280 + requests_issued: 1280 + result_summary.json: + duration_ns: 3704248419955.0 + ttft: + total: 5674433556480.0 + percentiles: + "50": 230893292.5 + "95": 330418109.5 + "90": 310513146.1 + output_sequence_lengths: + total: 7189576.0 + system_tps: 1940.9001 + system_desc.json: + tps_utilization: 0.686722 + results/system_c/qwen3-coder-480b/r16: + accuracy_results.json: {} + config.yaml: + concurrency: 16 + point.yaml: + concurrency: 16 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 16 + requests_completed: 160 + requests_issued: 160 + result_summary.json: + duration_ns: 9681657879146.0 + ttft: + total: 2364827467776.0 + percentiles: + "50": 96225076.0 + "95": 164579607.0 + "90": 150908700.8 + output_sequence_lengths: + total: 7193577.0 + system_tps: 743.0109 + system_desc.json: + tps_utilization: 0.262889 + results/system_c/qwen3-coder-480b/r256: + accuracy_results.json: {} + config.yaml: + concurrency: 256 + point.yaml: + concurrency: 256 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 256 + requests_completed: 2560 + requests_issued: 2560 + result_summary.json: + duration_ns: 3055408461497.0005 + ttft: + total: 8782727024640.0 + percentiles: + "50": 357370077.5 + "95": 457645674.25 + "90": 437590554.9 + output_sequence_lengths: + total: 7178996.0 + system_tps: 2349.6027 + system_desc.json: + tps_utilization: 0.831327 + results/system_c/qwen3-coder-480b/r32: + accuracy_results.json: {} + config.yaml: + concurrency: 32 + point.yaml: + concurrency: 32 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 32 + requests_completed: 320 + requests_issued: 320 + result_summary.json: + duration_ns: 7215663280145.0 + ttft: + total: 3557367164928.0 + percentiles: + "50": 144749640.5 + "95": 211749072.25 + "90": 198349185.9 + output_sequence_lengths: + total: 7193551.0 + system_tps: 996.9355 + system_desc.json: + tps_utilization: 0.352732 + results/system_c/qwen3-coder-480b/r512: + accuracy_results.json: {} + config.yaml: + concurrency: 512 + point.yaml: + concurrency: 512 + offline: elected + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 512 + requests_completed: 5120 + requests_issued: 5120 + result_summary.json: + duration_ns: 2550450936890.0 + ttft: + total: 13680033595392.0 + percentiles: + "50": 556641992.0 + "95": 841800473.5 + "90": 784768777.2 + output_sequence_lengths: + total: 7208408.0 + system_tps: 2826.3269 + system_desc.json: + tps_utilization: 1.0 + results/system_c/qwen3-coder-480b/r64: + accuracy_results.json: {} + config.yaml: + concurrency: 64 + point.yaml: + concurrency: 64 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 64 + requests_completed: 640 + requests_issued: 640 + result_summary.json: + duration_ns: 4947046368648.0 + ttft: + total: 3985956200448.0 + percentiles: + "50": 162188973.0 + "95": 255115666.5 + "90": 236530327.8 + output_sequence_lengths: + total: 7194216.0 + system_tps: 1454.2447 + system_desc.json: + tps_utilization: 0.514535 + results/system_c/qwen3-coder-480b/r8: + accuracy_results.json: {} + config.yaml: + concurrency: 8 + point.yaml: + concurrency: 8 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 8 + requests_completed: 80 + requests_issued: 80 + result_summary.json: + duration_ns: 17117165266230.002 + ttft: + total: 2276072361984.0 + percentiles: + "50": 92613621.5 + "95": 157459882.0 + "90": 144490629.9 + output_sequence_lengths: + total: 7185824.0 + system_tps: 419.8022 + system_desc.json: + tps_utilization: 0.148533 diff --git a/tests/fixtures/validation/cases/sub_d.yaml b/tests/fixtures/validation/cases/sub_d.yaml new file mode 100644 index 000000000..b7266fe3b --- /dev/null +++ b/tests/fixtures/validation/cases/sub_d.yaml @@ -0,0 +1,385 @@ +defaults: + accuracy_results.json: + humaneval: + dataset_name: humaneval + num_samples: 500 + score: 0.45 + extractor: ExactMatchExtractor + n_repeats: 1 + config.yaml: {} + point.yaml: + dataset: mlperf-perf-dataset-v1 + dataset_link: https://example.com/dataset + dataset_name: CNN/DailyMail + dataset_type: Performance + division: Standardized + link_to_model: https://example.com/model + link_to_model_transformation: https://example.com/quantization + max_supported_concurrency: 1024 + model_name: qwen3-coder-480b + model_notes: "" + model_precision: FP16 + runtime_settings: + load_pattern: concurrency + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true + seed_set: A + shared_docs: docs + shared_src: src + steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 + target_cohort: 2026-10-C1 + warmup: + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + result_summary.json: + n_samples_issued: 24576 + n_samples_completed: 24576 + n_samples_failed: 0 + output_sequence_lengths: + percentiles: {} + tpot: + total: 99508224000.00002 + percentiles: + "50": 4000000.0 + "90": 5500000.0 + "95": 6000000.0 + "99": 6500000.0 + tps_per_user: 181.8182 + system_desc.json: + submitter_org_names: Submitter-D + submitter_contact: contact@example.com + system_name: system_d + system_category: datacenter + max_supported_concurrency: 1024 + serving_framework: vLLM (TPU) + node_types: + - system_node_ensemble_id: 0 + number_of_nodes: 1 + host_processor_model_name: Custom TPU Host CPU + host_processors_per_node: 1 + host_processor_core_count: 24 + host_memory_capacity: 192 GB + host_networking: Ethernet 100Gb + host_storage_capacity: 10 TB + host_storage_type: NVMe SSD + operating_system: Debian 11 + host_memory_configuration: 8x 64GB DDR5 + host_network_card_count: 1x NIC + driver: "1.0" + filesystem: ext4 + accelerator_info: + - accelerator_model_name: Google TPU v5p + accelerators_per_node: 8 + accelerator_memory_capacity: 96 GB HBM2e + accelerator_memory_type: HBM3e + division: Standardized + model_precision: FP16 + link_to_model: https://example.com/model + dataset_id: cnn_dailymail + dataset_name: CNN/DailyMail + input_token_average: 870.0 + output_token_average: 128.0 + dataset_type: performance + dataset_link: https://example.com/dataset + measured_accuracy_score: "38.7" + system_size: 1 node + model_id: qwen3-coder-480b + system_node_ensemble_count: 1 + system_node_ensemble_total: 1 + publication_status: Available + node_config: 8x H100 test node + config_summary: + disaggregated: null + expert_parallel: 1 + tensor_parallel: 8 + pipeline_parallel: 1 + data_parallel: 1 + batch: null + cooling: Liquid-cooled +files: + docs/.gitkeep: "" + results/system_d/system_power.json: + system_desc_id: system_d + cooling: liquid + node_sets: + - node_set_id: 0 + system_node_ensemble_id: 0 + nodes_provisioned: 1 + power_method: component_sum + components: + cpu: + model: Custom TPU Host CPU + count_per_node: 1 + tdp_per_unit: + value_w: 350 + source_type: vendor_spec + source: https://example.com/cpu-spec + accelerator: + model: Google TPU v5p + count_per_node: 8 + tdp_per_unit: + value_w: 700 + source_type: vendor_spec + source: https://example.com/accelerator-spec + scale_up_network: + method: declared_tdp + switch_count: 1 + tdp_per_switch: + value_w: 3500 + source_type: vendor_spec + source: https://example.com/switch-spec + scale_out: + present: false + src/vllm/README.md: "# vllm + + + Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce + a point here. + + " +points: + results/system_d/qwen3-coder-480b/r1024: + accuracy_results.json: {} + config.yaml: + concurrency: 1024 + point.yaml: + concurrency: 1024 + offline: elected + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 1024 + requests_completed: 10240 + requests_issued: 10240 + result_summary.json: + duration_ns: 2448212914685.0 + ttft: + total: 19499568439296.0 + percentiles: + "50": 793439471.0 + "95": 1030084586.4999999 + "90": 982755563.3999999 + output_sequence_lengths: + total: 7186180.0 + system_tps: 2935.2758 + system_desc.json: + tps_utilization: 1.0 + results/system_d/qwen3-coder-480b/r128: + accuracy_results.json: {} + config.yaml: + concurrency: 128 + point.yaml: + concurrency: 128 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 128 + requests_completed: 1280 + requests_issued: 1280 + result_summary.json: + duration_ns: 4981652524410.999 + ttft: + total: 5200658755584.0 + percentiles: + "50": 211615346.5 + "95": 274691318.25 + "90": 262076123.9 + output_sequence_lengths: + total: 7208951.0 + system_tps: 1447.1003 + system_desc.json: + tps_utilization: 0.493003 + results/system_d/qwen3-coder-480b/r16: + accuracy_results.json: {} + config.yaml: + concurrency: 16 + point.yaml: + concurrency: 16 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 16 + requests_completed: 160 + requests_issued: 160 + result_summary.json: + duration_ns: 21458438875063.0 + ttft: + total: 3068488581120.0 + percentiles: + "50": 124857120.0 + "95": 181258587.5 + "90": 169978294.0 + output_sequence_lengths: + total: 7214114.0 + system_tps: 336.1901 + system_desc.json: + tps_utilization: 0.114534 + results/system_d/qwen3-coder-480b/r256: + accuracy_results.json: {} + config.yaml: + concurrency: 256 + point.yaml: + concurrency: 256 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 256 + requests_completed: 2560 + requests_issued: 2560 + result_summary.json: + duration_ns: 3735858007321.0 + ttft: + total: 7190722621440.0 + percentiles: + "50": 292591252.5 + "95": 370424039.25 + "90": 354857481.9 + output_sequence_lengths: + total: 7211649.0 + system_tps: 1930.3863 + system_desc.json: + tps_utilization: 0.657651 + results/system_d/qwen3-coder-480b/r32: + accuracy_results.json: {} + config.yaml: + concurrency: 32 + point.yaml: + concurrency: 32 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 32 + requests_completed: 320 + requests_issued: 320 + result_summary.json: + duration_ns: 12616634802170.0 + ttft: + total: 3350546350080.0 + percentiles: + "50": 136334080.0 + "95": 193898324.0 + "90": 182385475.2 + output_sequence_lengths: + total: 7189678.0 + system_tps: 569.857 + system_desc.json: + tps_utilization: 0.194141 + results/system_d/qwen3-coder-480b/r512: + accuracy_results.json: {} + config.yaml: + concurrency: 512 + point.yaml: + concurrency: 512 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 512 + requests_completed: 5120 + requests_issued: 5120 + result_summary.json: + duration_ns: 2852962195525.0 + ttft: + total: 10591377063936.0 + percentiles: + "50": 430964236.0 + "95": 646502413.0 + "90": 603394777.6 + output_sequence_lengths: + total: 7193097.0 + system_tps: 2521.2732 + system_desc.json: + tps_utilization: 0.858956 + results/system_d/qwen3-coder-480b/r64: + accuracy_results.json: {} + config.yaml: + concurrency: 64 + point.yaml: + concurrency: 64 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 64 + requests_completed: 640 + requests_issued: 640 + result_summary.json: + duration_ns: 7605866967850.0 + ttft: + total: 4213553332224.0 + percentiles: + "50": 171449924.0 + "95": 229212318.25 + "90": 217659839.4 + output_sequence_lengths: + total: 7198828.0 + system_tps: 946.4836 + system_desc.json: + tps_utilization: 0.322451 + results/system_d/qwen3-coder-480b/r8: + accuracy_results.json: {} + config.yaml: + concurrency: 8 + point.yaml: + concurrency: 8 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 8 + requests_completed: 80 + requests_issued: 80 + result_summary.json: + duration_ns: 40699327677953.0 + ttft: + total: 3014703452160.0 + percentiles: + "50": 122668597.5 + "95": 178853483.5 + "90": 167616506.3 + output_sequence_lengths: + total: 7176538.0 + system_tps: 176.3306 + system_desc.json: + tps_utilization: 0.060073 diff --git a/tests/fixtures/validation/cases/sub_e.yaml b/tests/fixtures/validation/cases/sub_e.yaml new file mode 100644 index 000000000..47c57f22b --- /dev/null +++ b/tests/fixtures/validation/cases/sub_e.yaml @@ -0,0 +1,453 @@ +defaults: + accuracy_results.json: + cnn_dailymail_validation: + dataset_name: cnn_dailymail_validation + num_samples: 13368 + score: + rouge1: 38.7792 + rouge2: 15.9075 + rougel: 24.4957 + rougelsum: 35.793 + gen_len: 8167644.0 + extractor: RougeExtractor + n_repeats: 1 + config.yaml: {} + point.yaml: + dataset: mlperf-perf-dataset-v1 + dataset_link: https://example.com/dataset + dataset_name: CNN/DailyMail + dataset_type: Performance + division: Standardized + link_to_model: https://example.com/model + link_to_model_transformation: https://example.com/quantization + max_supported_concurrency: 1024 + model_name: llama3-8b + model_notes: "" + model_precision: FP16 + runtime_settings: + load_pattern: concurrency + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true + seed_set: A + shared_docs: docs + shared_src: src + steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 + target_cohort: 2026-10-C1 + warmup: + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + result_summary.json: + n_samples_issued: 13368 + n_samples_completed: 13368 + n_samples_failed: 0 + output_sequence_lengths: + total: 1711102.0 + percentiles: {} + tpot: + total: 54127032000.00001 + percentiles: + "50": 4000000.0 + "90": 5500000.0 + "95": 6000000.0 + "99": 6500000.0 + tps_per_user: 181.8182 + system_desc.json: + submitter_org_names: Submitter-E + submitter_contact: contact@example.com + system_name: system_e + system_category: datacenter + max_supported_concurrency: 1024 + serving_framework: vLLM + node_types: + - system_node_ensemble_id: 0 + number_of_nodes: 1 + host_processor_model_name: Intel Xeon Platinum 8480+ + host_processors_per_node: 2 + host_processor_core_count: 60 + host_memory_capacity: 512 GB + host_networking: Ethernet 200Gb + host_storage_capacity: 15 TB + host_storage_type: NVMe SSD + operating_system: Ubuntu 22.04 + host_memory_configuration: 8x 64GB DDR5 + host_network_card_count: 1x NIC + driver: "1.0" + filesystem: ext4 + accelerator_info: + - accelerator_model_name: Intel Gaudi 3 + accelerators_per_node: 8 + accelerator_memory_capacity: 96 GB HBM2e + accelerator_memory_type: HBM3e + division: Standardized + model_precision: FP16 + link_to_model: https://example.com/model + dataset_id: cnn_dailymail + dataset_name: CNN/DailyMail + input_token_average: 870.0 + output_token_average: 128.0 + dataset_type: performance + dataset_link: https://example.com/dataset + measured_accuracy_score: "38.7" + system_size: 1 node + model_id: llama3-8b + system_node_ensemble_count: 1 + system_node_ensemble_total: 1 + publication_status: Available + node_config: 8x H100 test node + config_summary: + disaggregated: null + expert_parallel: 1 + tensor_parallel: 8 + pipeline_parallel: 1 + data_parallel: 1 + batch: null + cooling: Air-cooled +files: + docs/.gitkeep: "" + results/system_e/system_power.json: + system_desc_id: system_e + cooling: air + node_sets: + - node_set_id: 0 + system_node_ensemble_id: 0 + nodes_provisioned: 1 + power_method: component_sum + components: + cpu: + model: Intel Xeon Platinum 8480+ + count_per_node: 2 + tdp_per_unit: + value_w: 350 + source_type: vendor_spec + source: https://example.com/cpu-spec + accelerator: + model: Intel Gaudi 3 + count_per_node: 8 + tdp_per_unit: + value_w: 700 + source_type: vendor_spec + source: https://example.com/accelerator-spec + scale_up_network: + method: declared_tdp + switch_count: 1 + tdp_per_switch: + value_w: 3500 + source_type: vendor_spec + source: https://example.com/switch-spec + scale_out: + present: false + src/vllm/README.md: "# vllm + + + Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce + a point here. + + " +points: + results/system_e/llama3-8b/r1: + accuracy_results.json: {} + config.yaml: + concurrency: 1 + point.yaml: + concurrency: 1 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 1 + requests_completed: 10 + requests_issued: 10 + result_summary.json: + duration_ns: 25761392228349.0 + ttft: + total: 1364073025980.0 + percentiles: + "50": 102040172.5 + "95": 239159702.6 + "90": 211735796.57999998 + system_tps: 66.4212 + system_desc.json: + tps_utilization: 0.019947 + results/system_e/llama3-8b/r1024: + accuracy_results.json: {} + config.yaml: + concurrency: 1024 + point.yaml: + concurrency: 1024 + offline: elected + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 1024 + requests_completed: 10240 + requests_issued: 10240 + result_summary.json: + duration_ns: 515488552187.99994 + ttft: + total: 108213125776644.0 + percentiles: + "50": 8094937595.5 + "95": 17493931139.45 + "90": 15614132430.66 + system_tps: 3319.3792 + system_desc.json: + tps_utilization: 0.99684 + results/system_e/llama3-8b/r128: + accuracy_results.json: {} + config.yaml: + concurrency: 128 + point.yaml: + concurrency: 128 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 128 + requests_completed: 1280 + requests_issued: 1280 + result_summary.json: + duration_ns: 627339011078.0 + ttft: + total: 2788710324048.0 + percentiles: + "50": 208610886.0 + "95": 438744546.99999994 + "90": 392717814.79999995 + system_tps: 2727.5555 + system_desc.json: + tps_utilization: 0.81911 + results/system_e/llama3-8b/r16: + accuracy_results.json: {} + config.yaml: + concurrency: 16 + point.yaml: + concurrency: 16 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 16 + requests_completed: 160 + requests_issued: 160 + result_summary.json: + duration_ns: 2064726029363.0 + ttft: + total: 1680393412872.0 + percentiles: + "50": 125702679.0 + "95": 274517621.75 + "90": 244754633.2 + system_tps: 828.7308 + system_desc.json: + tps_utilization: 0.248875 + results/system_e/llama3-8b/r2: + accuracy_results.json: {} + config.yaml: + concurrency: 2 + point.yaml: + concurrency: 2 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 2 + requests_completed: 20 + requests_issued: 20 + result_summary.json: + duration_ns: 13163162954524.0 + ttft: + total: 1484351746344.0 + percentiles: + "50": 111037683.0 + "95": 243860930.6 + "90": 217296281.07999998 + system_tps: 129.9917 + system_desc.json: + tps_utilization: 0.039038 + results/system_e/llama3-8b/r256: + accuracy_results.json: {} + config.yaml: + concurrency: 256 + point.yaml: + concurrency: 256 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 256 + requests_completed: 2560 + requests_issued: 2560 + result_summary.json: + duration_ns: 543519373006.00006 + ttft: + total: 3961749503400.0 + percentiles: + "50": 296360675.0 + "95": 614244970.3 + "90": 550668111.24 + system_tps: 3148.1895 + system_desc.json: + tps_utilization: 0.94543 + results/system_e/llama3-8b/r32: + accuracy_results.json: {} + config.yaml: + concurrency: 32 + point.yaml: + concurrency: 32 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 32 + requests_completed: 320 + requests_issued: 320 + result_summary.json: + duration_ns: 1210809443938.0 + ttft: + total: 1830336280008.0 + percentiles: + "50": 136919231.0 + "95": 299232559.5 + "90": 266769893.8 + system_tps: 1413.1885 + system_desc.json: + tps_utilization: 0.424393 + results/system_e/llama3-8b/r4: + accuracy_results.json: {} + config.yaml: + concurrency: 4 + point.yaml: + concurrency: 4 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 4 + requests_completed: 40 + requests_issued: 40 + result_summary.json: + duration_ns: 6914982314847.001 + ttft: + total: 1477899507360.0 + percentiles: + "50": 110555020.0 + "95": 246967133.4 + "90": 219684710.72000003 + system_tps: 247.4485 + system_desc.json: + tps_utilization: 0.074311 + results/system_e/llama3-8b/r512: + accuracy_results.json: {} + config.yaml: + concurrency: 512 + point.yaml: + concurrency: 512 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 512 + requests_completed: 5120 + requests_issued: 5120 + result_summary.json: + duration_ns: 513859639398.0 + ttft: + total: 6792119454924.0 + percentiles: + "50": 508087930.5 + "95": 2020109026.3999984 + "90": 1717704807.2199988 + system_tps: 3329.9015 + system_desc.json: + tps_utilization: 1.0 + results/system_e/llama3-8b/r64: + accuracy_results.json: {} + config.yaml: + concurrency: 64 + point.yaml: + concurrency: 64 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 64 + requests_completed: 640 + requests_issued: 640 + result_summary.json: + duration_ns: 817907368671.0 + ttft: + total: 2259057999168.0 + percentiles: + "50": 168989976.0 + "95": 370875724.7999998 + "90": 330498575.03999984 + system_tps: 2092.0486 + system_desc.json: + tps_utilization: 0.628261 + results/system_e/llama3-8b/r8: + accuracy_results.json: {} + config.yaml: + concurrency: 8 + point.yaml: + concurrency: 8 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 8 + requests_completed: 80 + requests_issued: 80 + result_summary.json: + duration_ns: 3722968524345.0 + ttft: + total: 1551415687080.0 + percentiles: + "50": 116054435.0 + "95": 253079730.75 + "90": 225674671.60000002 + system_tps: 459.6069 + system_desc.json: + tps_utilization: 0.138024 diff --git a/tests/fixtures/validation/cases/sub_f.yaml b/tests/fixtures/validation/cases/sub_f.yaml new file mode 100644 index 000000000..38cfdd932 --- /dev/null +++ b/tests/fixtures/validation/cases/sub_f.yaml @@ -0,0 +1,471 @@ +defaults: + accuracy_results.json: + cnn_dailymail_validation: + dataset_name: cnn_dailymail_validation + num_samples: 13368 + score: + rouge1: 38.7792 + rouge2: 15.9075 + rougel: 24.4957 + rougelsum: 35.793 + gen_len: 8167644.0 + extractor: RougeExtractor + n_repeats: 1 + config.yaml: {} + point.yaml: + dataset: mlperf-perf-dataset-v1 + dataset_link: https://example.com/dataset + dataset_name: CNN/DailyMail + dataset_type: Performance + division: Standardized + link_to_model: https://example.com/model + link_to_model_transformation: https://example.com/quantization + max_supported_concurrency: 1024 + model_name: llama3-8b + model_notes: "" + model_precision: FP16 + runtime_settings: + load_pattern: concurrency + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true + seed_set: A + shared_docs: docs + shared_src: src + steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 + target_cohort: 2026-10-C1 + warmup: + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + result_summary.json: + n_samples_issued: 13368 + n_samples_completed: 13368 + n_samples_failed: 0 + output_sequence_lengths: + total: 1711102.0 + percentiles: {} + tpot: + total: 54127032000.00001 + percentiles: + "50": 4000000.0 + "90": 5500000.0 + "95": 6000000.0 + "99": 6500000.0 + tps_per_user: 181.8182 + system_desc.json: + submitter_org_names: Submitter-F + submitter_contact: contact@example.com + system_name: system_f + system_category: datacenter + max_supported_concurrency: 1024 + serving_framework: vLLM + node_types: + - system_node_ensemble_id: 0 + number_of_nodes: 2 + host_processor_model_name: Intel Xeon Platinum 8480+ + host_processors_per_node: 2 + host_processor_core_count: 60 + host_memory_capacity: 512 GB + host_networking: Ethernet 200Gb + host_storage_capacity: 15 TB + host_storage_type: NVMe SSD + operating_system: Ubuntu 22.04 + host_memory_configuration: 8x 64GB DDR5 + host_network_card_count: 1x NIC + driver: "1.0" + filesystem: ext4 + accelerator_info: + - accelerator_model_name: Intel Gaudi 3 + accelerators_per_node: 8 + accelerator_memory_capacity: 96 GB HBM2e + accelerator_memory_type: HBM3e + division: Standardized + model_precision: FP16 + link_to_model: https://example.com/model + dataset_id: cnn_dailymail + dataset_name: CNN/DailyMail + input_token_average: 870.0 + output_token_average: 128.0 + dataset_type: performance + dataset_link: https://example.com/dataset + measured_accuracy_score: "38.7" + system_size: 1 node + model_id: llama3-8b + system_node_ensemble_count: 1 + system_node_ensemble_total: 2 + publication_status: Available + node_config: 8x H100 test node + config_summary: + disaggregated: null + expert_parallel: 1 + tensor_parallel: 8 + pipeline_parallel: 1 + data_parallel: 2 + batch: null + cooling: Air-cooled +files: + docs/.gitkeep: "" + results/system_f/system_power.json: + system_desc_id: system_f + cooling: air + node_sets: + - node_set_id: 0 + system_node_ensemble_id: 0 + nodes_provisioned: 2 + power_method: component_sum + components: + cpu: + model: Intel Xeon Platinum 8480+ + count_per_node: 2 + tdp_per_unit: + value_w: 350 + source_type: vendor_spec + source: https://example.com/cpu-spec + accelerator: + model: Intel Gaudi 3 + count_per_node: 8 + tdp_per_unit: + value_w: 700 + source_type: vendor_spec + source: https://example.com/accelerator-spec + scale_up_network: + method: declared_tdp + switch_count: 1 + tdp_per_switch: + value_w: 3500 + source_type: vendor_spec + source: https://example.com/switch-spec + scale_out: + present: true + cabling: passive + required_bandwidth_tbps: 6.4 + nics: + count_per_node: 8 + bandwidth_per_nic_gbps: 400 + counted: true + tdp_per_nic: + value_w: 25 + source_type: vendor_spec + source: https://example.com/nic-spec + switches: + - model: NVIDIA Spectrum SN5610 + count: 1 + bandwidth_tbps: 51.2 + power_per_switch: + value_w: 900 + source_type: vendor_spec + source: https://example.com/switch-spec + src/vllm/README.md: "# vllm + + + Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce + a point here. + + " +points: + results/system_f/llama3-8b/r1: + accuracy_results.json: {} + config.yaml: + concurrency: 1 + point.yaml: + concurrency: 1 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 1 + requests_completed: 10 + requests_issued: 10 + result_summary.json: + duration_ns: 45583460351935.0 + ttft: + total: 2574627124512.0 + percentiles: + "50": 192596284.0 + "95": 449459125.5 + "90": 398086557.20000005 + system_tps: 37.5378 + system_desc.json: + tps_utilization: 0.022095 + results/system_f/llama3-8b/r1024: + accuracy_results.json: {} + config.yaml: + concurrency: 1024 + point.yaml: + concurrency: 1024 + offline: elected + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 1024 + requests_completed: 10240 + requests_issued: 10240 + result_summary.json: + duration_ns: 1011902488209.0 + ttft: + total: 479444018825568.0 + percentiles: + "50": 35865052276.0 + "95": 46774478776.74999 + "90": 44592593476.59999 + system_tps: 1690.9752 + system_desc.json: + tps_utilization: 0.995315 + results/system_f/llama3-8b/r128: + accuracy_results.json: {} + config.yaml: + concurrency: 128 + point.yaml: + concurrency: 128 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 128 + requests_completed: 1280 + requests_issued: 1280 + result_summary.json: + duration_ns: 1192600590769.0 + ttft: + total: 5379052027176.0 + percentiles: + "50": 402382707.0 + "95": 869390356.9 + "90": 775988826.9200001 + system_tps: 1434.7653 + system_desc.json: + tps_utilization: 0.844509 + results/system_f/llama3-8b/r16: + accuracy_results.json: {} + config.yaml: + concurrency: 16 + point.yaml: + concurrency: 16 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 16 + requests_completed: 160 + requests_issued: 160 + result_summary.json: + duration_ns: 3505224837302.0005 + ttft: + total: 3373550899608.0 + percentiles: + "50": 252360181.0 + "95": 550608223.3999999 + "90": 490958614.9199999 + system_tps: 488.1576 + system_desc.json: + tps_utilization: 0.287332 + results/system_f/llama3-8b/r2: + accuracy_results.json: {} + config.yaml: + concurrency: 2 + point.yaml: + concurrency: 2 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 2 + requests_completed: 20 + requests_issued: 20 + result_summary.json: + duration_ns: 23023578002485.996 + ttft: + total: 2892442942776.0 + percentiles: + "50": 216370657.0 + "95": 457229317.34999996 + "90": 409057585.28 + system_tps: 74.3196 + system_desc.json: + tps_utilization: 0.043745 + results/system_f/llama3-8b/r256: + accuracy_results.json: {} + config.yaml: + concurrency: 256 + point.yaml: + concurrency: 256 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 256 + requests_completed: 2560 + requests_issued: 2560 + result_summary.json: + duration_ns: 1045116285522.0 + ttft: + total: 7725994840968.0 + percentiles: + "50": 577946951.0 + "95": 1242402278.2499998 + "90": 1109511212.7999997 + system_tps: 1637.236 + system_desc.json: + tps_utilization: 0.963684 + results/system_f/llama3-8b/r32: + accuracy_results.json: {} + config.yaml: + concurrency: 32 + point.yaml: + concurrency: 32 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 32 + requests_completed: 320 + requests_issued: 320 + result_summary.json: + duration_ns: 2182110754084.9998 + ttft: + total: 3596106784332.0 + percentiles: + "50": 269008586.5 + "95": 604284570.0 + "90": 537229373.3 + system_tps: 784.1499 + system_desc.json: + tps_utilization: 0.461554 + results/system_f/llama3-8b/r4: + accuracy_results.json: {} + config.yaml: + concurrency: 4 + point.yaml: + concurrency: 4 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 4 + requests_completed: 40 + requests_issued: 40 + result_summary.json: + duration_ns: 11622328296428.0 + ttft: + total: 2922014582988.0 + percentiles: + "50": 218582778.5 + "95": 470498090.7499999 + "90": 420115028.29999995 + system_tps: 147.2254 + system_desc.json: + tps_utilization: 0.086657 + results/system_f/llama3-8b/r512: + accuracy_results.json: {} + config.yaml: + concurrency: 512 + point.yaml: + concurrency: 512 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 512 + requests_completed: 5120 + requests_issued: 5120 + result_summary.json: + duration_ns: 1007161846898.0 + ttft: + total: 19342596948528.0 + percentiles: + "50": 1446932746.0 + "95": 17416985904.85 + "90": 14222975273.08 + system_tps: 1698.9345 + system_desc.json: + tps_utilization: 1.0 + results/system_f/llama3-8b/r64: + accuracy_results.json: {} + config.yaml: + concurrency: 64 + point.yaml: + concurrency: 64 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 64 + requests_completed: 640 + requests_issued: 640 + result_summary.json: + duration_ns: 1528920443390.0 + ttft: + total: 4340911548228.0 + percentiles: + "50": 324724083.5 + "95": 700762596.05 + "90": 625554893.54 + system_tps: 1119.157 + system_desc.json: + tps_utilization: 0.658741 + results/system_f/llama3-8b/r8: + accuracy_results.json: {} + config.yaml: + concurrency: 8 + point.yaml: + concurrency: 8 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 8 + requests_completed: 80 + requests_issued: 80 + result_summary.json: + duration_ns: 6211960687374.0 + ttft: + total: 3152037458208.0 + percentiles: + "50": 235789756.0 + "95": 509306917.05 + "90": 454603484.84000003 + system_tps: 275.4528 + system_desc.json: + tps_utilization: 0.162133 diff --git a/tests/fixtures/validation/cases/sub_g.yaml b/tests/fixtures/validation/cases/sub_g.yaml new file mode 100644 index 000000000..f6024eade --- /dev/null +++ b/tests/fixtures/validation/cases/sub_g.yaml @@ -0,0 +1,394 @@ +defaults: + accuracy_results.json: + cnn_dailymail_validation: + dataset_name: cnn_dailymail_validation + num_samples: 500 + score: 0.45 + extractor: RougeExtractor + n_repeats: 1 + config.yaml: {} + point.yaml: + dataset: mlperf-perf-dataset-v1 + dataset_link: https://example.com/dataset + dataset_name: CNN/DailyMail + dataset_type: Performance + division: Standardized + link_to_model: https://example.com/model + link_to_model_transformation: https://example.com/quantization + max_supported_concurrency: 2048 + model_name: llama3-70b + model_notes: "" + model_precision: FP16 + runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true + seed_set: A + shared_docs: docs + shared_src: src + steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 + target_cohort: 2026-10-C1 + warmup: + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + result_summary.json: + n_samples_issued: 24576 + n_samples_completed: 24576 + n_samples_failed: 0 + output_sequence_lengths: + percentiles: {} + tpot: + total: 99508224000.00002 + percentiles: + "50": 4000000.0 + "90": 5500000.0 + "95": 6000000.0 + "99": 6500000.0 + tps_per_user: 181.8182 + system_desc.json: + submitter_org_names: Submitter-G + submitter_contact: contact@example.com + system_name: system_g + system_category: datacenter + max_supported_concurrency: 2048 + serving_framework: vLLM + node_types: + - system_node_ensemble_id: 0 + number_of_nodes: 1 + host_processor_model_name: Intel Xeon Platinum 8480+ + host_processors_per_node: 2 + host_processor_core_count: 60 + host_memory_capacity: 2 TiB + host_networking: InfiniBand NDR + host_storage_capacity: 15 TB + host_storage_type: NVMe SSD + operating_system: Ubuntu 22.04 + host_memory_configuration: 8x 64GB DDR5 + host_network_card_count: 1x NIC + driver: "1.0" + filesystem: ext4 + accelerator_info: + - accelerator_model_name: NVIDIA H100-SXM5-80GB + accelerators_per_node: 8 + accelerator_memory_capacity: 80 GB HBM3 + accelerator_memory_type: HBM3e + accelerator_interconnect: NVLink 4.0 900 GB/s + division: Standardized + model_precision: FP16 + link_to_model: https://example.com/model + dataset_id: cnn_dailymail + dataset_name: CNN/DailyMail + input_token_average: 870.0 + output_token_average: 128.0 + dataset_type: performance + dataset_link: https://example.com/dataset + measured_accuracy_score: "38.7" + system_size: 1 node + model_id: llama3-70b + system_node_ensemble_count: 1 + system_node_ensemble_total: 1 + publication_status: Available + node_config: 8x H100 test node + config_summary: + disaggregated: null + expert_parallel: 1 + tensor_parallel: 8 + pipeline_parallel: 1 + data_parallel: 1 + batch: null + cooling: Air-cooled +files: + docs/.gitkeep: "" + results/system_g/system_power.json: + system_desc_id: system_g + cooling: air + node_sets: + - node_set_id: 0 + system_node_ensemble_id: 0 + nodes_provisioned: 1 + power_method: component_sum + components: + cpu: + model: Intel Xeon Platinum 8480+ + count_per_node: 2 + tdp_per_unit: + value_w: 350 + source_type: vendor_spec + source: https://example.com/cpu-spec + accelerator: + model: NVIDIA H100-SXM5-80GB + count_per_node: 8 + tdp_per_unit: + value_w: 700 + source_type: vendor_spec + source: https://example.com/accelerator-spec + scale_up_network: + method: declared_tdp + switch_count: 1 + tdp_per_switch: + value_w: 3500 + source_type: vendor_spec + source: https://example.com/switch-spec + scale_out: + present: false + src/vllm/README.md: "# vllm + + + Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce + a point here. + + " +points: + results/system_g/llama3-70b/r1024: + accuracy_results.json: {} + config.yaml: + concurrency: 1024 + point.yaml: + concurrency: 1024 + warmup: + concurrency: 1024 + requests_completed: 10240 + requests_issued: 10240 + result_summary.json: + duration_ns: 358433737274.0 + ttft: + total: 5094461190144.0 + percentiles: + "50": 207294156.5 + "95": 475022456.0 + "90": 421476796.1 + output_sequence_lengths: + total: 5972600.0 + system_tps: 16663.052 + system_desc.json: + tps_utilization: 0.979024 + results/system_g/llama3-70b/r128: + accuracy_results.json: {} + config.yaml: + concurrency: 128 + point.yaml: + concurrency: 128 + warmup: + concurrency: 128 + requests_completed: 1280 + requests_issued: 1280 + result_summary.json: + duration_ns: 1154776483445.0 + ttft: + total: 2436995543040.0 + percentiles: + "50": 99161602.5 + "95": 649174965.25 + "90": 539172292.7 + output_sequence_lengths: + total: 5954161.0 + system_tps: 5156.1156 + system_desc.json: + tps_utilization: 0.302943 + results/system_g/llama3-70b/r1536: + accuracy_results.json: {} + config.yaml: + concurrency: 1536 + point.yaml: + concurrency: 1536 + warmup: + concurrency: 1536 + requests_completed: 15360 + requests_issued: 15360 + result_summary.json: + duration_ns: 371427380557.0 + ttft: + total: 6655861899264.0 + percentiles: + "50": 270827714.0 + "95": 2812902801.5 + "90": 2304487784.0 + output_sequence_lengths: + total: 5972600.0 + system_tps: 16080.1285 + system_desc.json: + tps_utilization: 0.944775 + results/system_g/llama3-70b/r192: + accuracy_results.json: {} + config.yaml: + concurrency: 192 + point.yaml: + concurrency: 192 + warmup: + concurrency: 192 + requests_completed: 1920 + requests_issued: 1920 + result_summary.json: + duration_ns: 1008814134637.0 + ttft: + total: 2930111361024.0 + percentiles: + "50": 119226536.5 + "95": 610194947.75 + "90": 512001265.5 + output_sequence_lengths: + total: 5970998.0 + system_tps: 5918.8286 + system_desc.json: + tps_utilization: 0.347756 + results/system_g/llama3-70b/r2048: + accuracy_results.json: {} + config.yaml: + concurrency: 2048 + point.yaml: + concurrency: 2048 + offline: elected + warmup: + concurrency: 2048 + requests_completed: 20480 + requests_issued: 20480 + result_summary.json: + duration_ns: 350915227928.0 + ttft: + total: 7688238305280.0 + percentiles: + "50": 312835217.5 + "95": 4378115873.25 + "90": 3565059742.1000004 + output_sequence_lengths: + total: 5972600.0 + system_tps: 17020.065 + system_desc.json: + tps_utilization: 1.0 + results/system_g/llama3-70b/r256: + accuracy_results.json: {} + config.yaml: + concurrency: 256 + point.yaml: + concurrency: 256 + warmup: + concurrency: 256 + requests_completed: 2560 + requests_issued: 2560 + result_summary.json: + duration_ns: 776418597591.0 + ttft: + total: 3292125204480.0 + percentiles: + "50": 133956917.5 + "95": 590184585.25 + "90": 498939051.70000005 + output_sequence_lengths: + total: 5962724.0 + system_tps: 7679.78 + system_desc.json: + tps_utilization: 0.451219 + results/system_g/llama3-70b/r384: + accuracy_results.json: {} + config.yaml: + concurrency: 384 + point.yaml: + concurrency: 384 + warmup: + concurrency: 384 + requests_completed: 3840 + requests_issued: 3840 + result_summary.json: + duration_ns: 517816400500.0 + ttft: + total: 3370839257088.0 + percentiles: + "50": 137159800.5 + "95": 434521706.0 + "90": 375049324.9 + output_sequence_lengths: + total: 5964063.0 + system_tps: 11517.7175 + system_desc.json: + tps_utilization: 0.676714 + results/system_g/llama3-70b/r512: + accuracy_results.json: {} + config.yaml: + concurrency: 512 + point.yaml: + concurrency: 512 + warmup: + concurrency: 512 + requests_completed: 5120 + requests_issued: 5120 + result_summary.json: + duration_ns: 444594042812.0 + ttft: + total: 3720757702656.0 + percentiles: + "50": 151398018.5 + "95": 409875781.5 + "90": 358180228.9 + output_sequence_lengths: + total: 5961100.0 + system_tps: 13407.9619 + system_desc.json: + tps_utilization: 0.787774 + results/system_g/llama3-70b/r64: + accuracy_results.json: {} + config.yaml: + concurrency: 64 + point.yaml: + concurrency: 64 + warmup: + concurrency: 64 + requests_completed: 640 + requests_issued: 640 + result_summary.json: + duration_ns: 2145983999331.0002 + ttft: + total: 2184279601152.0 + percentiles: + "50": 88878564.5 + "95": 958562205.25 + "90": 784625477.1 + output_sequence_lengths: + total: 5979513.0 + system_tps: 2786.3735 + system_desc.json: + tps_utilization: 0.163711 + results/system_g/llama3-70b/r768: + accuracy_results.json: {} + config.yaml: + concurrency: 768 + point.yaml: + concurrency: 768 + warmup: + concurrency: 768 + requests_completed: 7680 + requests_issued: 7680 + result_summary.json: + duration_ns: 388271410974.0 + ttft: + total: 4313087397888.0 + percentiles: + "50": 175499975.5 + "95": 426849699.25 + "90": 376579754.5 + output_sequence_lengths: + total: 5972600.0 + system_tps: 15382.5387 + system_desc.json: + tps_utilization: 0.903788 diff --git a/tests/fixtures/validation/cases/sub_h.yaml b/tests/fixtures/validation/cases/sub_h.yaml new file mode 100644 index 000000000..5b7c8e487 --- /dev/null +++ b/tests/fixtures/validation/cases/sub_h.yaml @@ -0,0 +1,394 @@ +defaults: + accuracy_results.json: + cnn_dailymail_validation: + dataset_name: cnn_dailymail_validation + num_samples: 500 + score: 0.45 + extractor: RougeExtractor + n_repeats: 1 + config.yaml: {} + point.yaml: + dataset: mlperf-perf-dataset-v1 + dataset_link: https://example.com/dataset + dataset_name: CNN/DailyMail + dataset_type: Performance + division: Standardized + link_to_model: https://example.com/model + link_to_model_transformation: https://example.com/quantization + max_supported_concurrency: 2048 + model_name: llama3-70b + model_notes: "" + model_precision: FP16 + runtime_settings: + load_pattern: concurrency + min_duration_ms: 1200000 + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true + seed_set: A + shared_docs: docs + shared_src: src + steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + duration_s: 1200.0 + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 + target_cohort: 2026-10-C1 + warmup: + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + result_summary.json: + n_samples_issued: 24576 + n_samples_completed: 24576 + n_samples_failed: 0 + output_sequence_lengths: + percentiles: {} + tpot: + total: 99508224000.00002 + percentiles: + "50": 4000000.0 + "90": 5500000.0 + "95": 6000000.0 + "99": 6500000.0 + tps_per_user: 181.8182 + system_desc.json: + submitter_org_names: Submitter-H + submitter_contact: contact@example.com + system_name: system_h + system_category: datacenter + max_supported_concurrency: 2048 + serving_framework: SGLang + node_types: + - system_node_ensemble_id: 0 + number_of_nodes: 1 + host_processor_model_name: Intel Xeon Platinum 8480+ + host_processors_per_node: 2 + host_processor_core_count: 60 + host_memory_capacity: 2 TiB + host_networking: InfiniBand NDR + host_storage_capacity: 15 TB + host_storage_type: NVMe SSD + operating_system: Ubuntu 22.04 + host_memory_configuration: 8x 64GB DDR5 + host_network_card_count: 1x NIC + driver: "1.0" + filesystem: ext4 + accelerator_info: + - accelerator_model_name: NVIDIA H100-SXM5-80GB + accelerators_per_node: 8 + accelerator_memory_capacity: 80 GB HBM3 + accelerator_memory_type: HBM3e + accelerator_interconnect: NVLink 4.0 900 GB/s + division: Standardized + model_precision: FP16 + link_to_model: https://example.com/model + dataset_id: cnn_dailymail + dataset_name: CNN/DailyMail + input_token_average: 870.0 + output_token_average: 128.0 + dataset_type: performance + dataset_link: https://example.com/dataset + measured_accuracy_score: "38.7" + system_size: 1 node + model_id: llama3-70b + system_node_ensemble_count: 1 + system_node_ensemble_total: 1 + publication_status: Available + node_config: 8x H100 test node + config_summary: + disaggregated: null + expert_parallel: 1 + tensor_parallel: 8 + pipeline_parallel: 1 + data_parallel: 1 + batch: null + cooling: Air-cooled +files: + docs/.gitkeep: "" + results/system_h/system_power.json: + system_desc_id: system_h + cooling: air + node_sets: + - node_set_id: 0 + system_node_ensemble_id: 0 + nodes_provisioned: 1 + power_method: component_sum + components: + cpu: + model: Intel Xeon Platinum 8480+ + count_per_node: 2 + tdp_per_unit: + value_w: 350 + source_type: vendor_spec + source: https://example.com/cpu-spec + accelerator: + model: NVIDIA H100-SXM5-80GB + count_per_node: 8 + tdp_per_unit: + value_w: 700 + source_type: vendor_spec + source: https://example.com/accelerator-spec + scale_up_network: + method: declared_tdp + switch_count: 1 + tdp_per_switch: + value_w: 3500 + source_type: vendor_spec + source: https://example.com/switch-spec + scale_out: + present: false + src/sglang/README.md: "# sglang + + + Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce + a point here. + + " +points: + results/system_h/llama3-70b/r1024: + accuracy_results.json: {} + config.yaml: + concurrency: 1024 + point.yaml: + concurrency: 1024 + warmup: + concurrency: 1024 + requests_completed: 10240 + requests_issued: 10240 + result_summary.json: + duration_ns: 392249945920.0 + ttft: + total: 3834814304256.0 + percentiles: + "50": 156038993.5 + "95": 510605960.25 + "90": 439692566.90000004 + output_sequence_lengths: + total: 5958417.0 + system_tps: 15190.3577 + system_desc.json: + tps_utilization: 1.0 + results/system_h/llama3-70b/r128: + accuracy_results.json: {} + config.yaml: + concurrency: 128 + point.yaml: + concurrency: 128 + warmup: + concurrency: 128 + requests_completed: 1280 + requests_issued: 1280 + result_summary.json: + duration_ns: 1403030676854.0 + ttft: + total: 2718343397376.0 + percentiles: + "50": 110609676.0 + "95": 758335350.75 + "90": 628790215.8 + output_sequence_lengths: + total: 5949986.0 + system_tps: 4240.8096 + system_desc.json: + tps_utilization: 0.279178 + results/system_h/llama3-70b/r1536: + accuracy_results.json: {} + config.yaml: + concurrency: 1536 + point.yaml: + concurrency: 1536 + warmup: + concurrency: 1536 + requests_completed: 15360 + requests_issued: 15360 + result_summary.json: + duration_ns: 402038739154.0 + ttft: + total: 4694270042112.0 + percentiles: + "50": 191010337.0 + "95": 2130321446.0 + "90": 1742459224.2 + output_sequence_lengths: + total: 5958417.0 + system_tps: 14820.5046 + system_desc.json: + tps_utilization: 0.975652 + results/system_h/llama3-70b/r192: + accuracy_results.json: {} + config.yaml: + concurrency: 192 + point.yaml: + concurrency: 192 + warmup: + concurrency: 192 + requests_completed: 1920 + requests_issued: 1920 + result_summary.json: + duration_ns: 1001832049974.0001 + ttft: + total: 2768901292032.0 + percentiles: + "50": 112666882.0 + "95": 635672662.0 + "90": 531071506.0 + output_sequence_lengths: + total: 5956127.0 + system_tps: 5945.235 + system_desc.json: + tps_utilization: 0.391382 + results/system_h/llama3-70b/r2048: + accuracy_results.json: {} + config.yaml: + concurrency: 2048 + point.yaml: + concurrency: 2048 + offline: elected + warmup: + concurrency: 2048 + requests_completed: 20480 + requests_issued: 20480 + result_summary.json: + duration_ns: 409541274676.0 + ttft: + total: 5435139072000.0 + percentiles: + "50": 221156375.0 + "95": 4408367949.0 + "90": 3570925634.2000003 + output_sequence_lengths: + total: 5958417.0 + system_tps: 14549.0024 + system_desc.json: + tps_utilization: 0.957779 + results/system_h/llama3-70b/r256: + accuracy_results.json: {} + config.yaml: + concurrency: 256 + point.yaml: + concurrency: 256 + warmup: + concurrency: 256 + requests_completed: 2560 + requests_issued: 2560 + result_summary.json: + duration_ns: 799717038355.0 + ttft: + total: 2881879240704.0 + percentiles: + "50": 117263966.5 + "95": 558828541.0 + "90": 470515626.1 + output_sequence_lengths: + total: 5960490.0 + system_tps: 7453.2487 + system_desc.json: + tps_utilization: 0.490657 + results/system_h/llama3-70b/r384: + accuracy_results.json: {} + config.yaml: + concurrency: 384 + point.yaml: + concurrency: 384 + warmup: + concurrency: 384 + requests_completed: 3840 + requests_issued: 3840 + result_summary.json: + duration_ns: 600137480188.0 + ttft: + total: 3042139607040.0 + percentiles: + "50": 123784977.5 + "95": 483974886.0 + "90": 411936904.3 + output_sequence_lengths: + total: 5958391.0 + system_tps: 9928.3767 + system_desc.json: + tps_utilization: 0.653597 + results/system_h/llama3-70b/r512: + accuracy_results.json: {} + config.yaml: + concurrency: 512 + point.yaml: + concurrency: 512 + warmup: + concurrency: 512 + requests_completed: 5120 + requests_issued: 5120 + result_summary.json: + duration_ns: 500684255434.00006 + ttft: + total: 3142710755328.0 + percentiles: + "50": 127877228.0 + "95": 443205541.0 + "90": 380139878.4 + output_sequence_lengths: + total: 5958417.0 + system_tps: 11900.548 + system_desc.json: + tps_utilization: 0.783428 + results/system_h/llama3-70b/r64: + accuracy_results.json: {} + config.yaml: + concurrency: 64 + point.yaml: + concurrency: 64 + warmup: + concurrency: 64 + requests_completed: 640 + requests_issued: 640 + result_summary.json: + duration_ns: 2608958278520.0 + ttft: + total: 2562863296512.0 + percentiles: + "50": 104283174.5 + "95": 1100174349.75 + "90": 900996114.7 + output_sequence_lengths: + total: 5965062.0 + system_tps: 2286.3769 + system_desc.json: + tps_utilization: 0.150515 + results/system_h/llama3-70b/r768: + accuracy_results.json: {} + config.yaml: + concurrency: 768 + point.yaml: + concurrency: 768 + warmup: + concurrency: 768 + requests_completed: 7680 + requests_issued: 7680 + result_summary.json: + duration_ns: 513149916081.0 + ttft: + total: 3919180652544.0 + percentiles: + "50": 159471869.0 + "95": 529036573.0 + "90": 455123632.2 + output_sequence_lengths: + total: 5958417.0 + system_tps: 11611.4547 + system_desc.json: + tps_utilization: 0.764396 diff --git a/tests/fixtures/validation/cases/sub_i.yaml b/tests/fixtures/validation/cases/sub_i.yaml new file mode 100644 index 000000000..64d94ab96 --- /dev/null +++ b/tests/fixtures/validation/cases/sub_i.yaml @@ -0,0 +1,481 @@ +defaults: + accuracy_results.json: + math500: + dataset_name: math500 + num_samples: 4388 + score: + exact_match: 81.3582 + extractor: BoxedAnswerExtractor + n_repeats: 1 + config.yaml: {} + point.yaml: + dataset: mlperf-perf-dataset-v1 + dataset_link: https://example.com/dataset + dataset_name: CNN/DailyMail + dataset_type: Performance + division: Standardized + link_to_model: https://example.com/model + link_to_model_transformation: https://example.com/quantization + max_supported_concurrency: 512 + model_name: deepseek-r1 + model_notes: "" + model_precision: FP16 + runtime_settings: + load_pattern: concurrency + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true + seed_set: A + shared_docs: docs + shared_src: src + steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 + target_cohort: 2026-10-C1 + warmup: + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + result_summary.json: + n_samples_failed: 0 + output_sequence_lengths: + percentiles: {} + tpot: + percentiles: + "50": 4000000.0 + "90": 5500000.0 + "95": 6000000.0 + "99": 6500000.0 + tps_per_user: 181.8182 + system_desc.json: + submitter_org_names: Submitter-I + submitter_contact: contact@example.com + system_name: system_i + system_category: datacenter + max_supported_concurrency: 512 + serving_framework: TensorRT-LLM + node_types: + - system_node_ensemble_id: 0 + number_of_nodes: 1 + host_processor_model_name: Intel Xeon Platinum 8562Y+ + host_processors_per_node: 2 + host_processor_core_count: 32 + host_memory_capacity: 2 TiB + host_networking: InfiniBand NDR + host_storage_capacity: 15 TB + host_storage_type: NVMe SSD + operating_system: Ubuntu 22.04 + host_memory_configuration: 8x 64GB DDR5 + host_network_card_count: 1x NIC + driver: "1.0" + filesystem: ext4 + accelerator_info: + - accelerator_model_name: NVIDIA H200-SXM-141GB + accelerators_per_node: 8 + accelerator_memory_capacity: 141 GB HBM3e + accelerator_memory_type: HBM3e + accelerator_interconnect: 18x NVLink 4th Gen 900 GB/s + accelerator_host_interconnect: PCIe Gen5 + division: Standardized + model_precision: FP16 + link_to_model: https://example.com/model + dataset_id: cnn_dailymail + dataset_name: CNN/DailyMail + input_token_average: 870.0 + output_token_average: 128.0 + dataset_type: performance + dataset_link: https://example.com/dataset + measured_accuracy_score: "38.7" + system_size: 1 node + model_id: deepseek-r1 + system_node_ensemble_count: 1 + system_node_ensemble_total: 1 + publication_status: Available + node_config: 8x H100 test node + config_summary: + disaggregated: null + expert_parallel: 1 + tensor_parallel: 8 + pipeline_parallel: 1 + data_parallel: 1 + batch: null + cooling: Air-cooled +files: + docs/.gitkeep: "" + results/system_i/system_power.json: + system_desc_id: system_i + cooling: air + node_sets: + - node_set_id: 0 + system_node_ensemble_id: 0 + nodes_provisioned: 1 + power_method: component_sum + components: + cpu: + model: Intel Xeon Platinum 8562Y+ + count_per_node: 2 + tdp_per_unit: + value_w: 350 + source_type: vendor_spec + source: https://example.com/cpu-spec + accelerator: + model: NVIDIA H200-SXM-141GB + count_per_node: 8 + tdp_per_unit: + value_w: 700 + source_type: vendor_spec + source: https://example.com/accelerator-spec + scale_up_network: + method: declared_tdp + switch_count: 1 + tdp_per_switch: + value_w: 3500 + source_type: vendor_spec + source: https://example.com/switch-spec + scale_out: + present: false + src/tensorrt-llm/README.md: "# tensorrt-llm + + + Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce + a point here. + + " +points: + results/system_i/deepseek-r1/r1: + accuracy_results.json: {} + config.yaml: + concurrency: 1 + point.yaml: + concurrency: 1 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 1 + requests_completed: 10 + requests_issued: 10 + result_summary.json: + n_samples_issued: 29 + n_samples_completed: 29 + duration_ns: 1810326781531.0 + ttft: + total: 3239905752.0 + percentiles: + "50": 111720888.0 + "95": 121428535.79999998 + "90": 119487006.23999998 + output_sequence_lengths: + total: 168384.0 + tpot: + total: 117421000.00000001 + system_tps: 93.013 + system_desc.json: + tps_utilization: 0.01422 + results/system_i/deepseek-r1/r128: + accuracy_results.json: {} + config.yaml: + concurrency: 128 + point.yaml: + concurrency: 128 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 128 + requests_completed: 1280 + requests_issued: 1280 + result_summary.json: + n_samples_issued: 1675 + n_samples_completed: 1675 + duration_ns: 2363311856036.0 + ttft: + total: 268304677475.0 + percentiles: + "50": 160181897.0 + "95": 456279775.1 + "90": 397060199.48 + output_sequence_lengths: + total: 6215384.0 + tpot: + total: 6782075000.000001 + system_tps: 2629.9466 + system_desc.json: + tps_utilization: 0.402063 + results/system_i/deepseek-r1/r16: + accuracy_results.json: {} + config.yaml: + concurrency: 16 + point.yaml: + concurrency: 16 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 16 + requests_completed: 160 + requests_issued: 160 + result_summary.json: + n_samples_issued: 339 + n_samples_completed: 339 + duration_ns: 2003156043150.0 + ttft: + total: 45912856545.0 + percentiles: + "50": 135436155.0 + "95": 241107175.0999999 + "90": 219972971.07999992 + output_sequence_lengths: + total: 1378214.0 + tpot: + total: 1372611000.0000002 + system_tps: 688.0213 + system_desc.json: + tps_utilization: 0.105184 + results/system_i/deepseek-r1/r2: + accuracy_results.json: {} + config.yaml: + concurrency: 2 + point.yaml: + concurrency: 2 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 2 + requests_completed: 20 + requests_issued: 20 + result_summary.json: + n_samples_issued: 68 + n_samples_completed: 68 + duration_ns: 1944103059102.0002 + ttft: + total: 8684697316.0 + percentiles: + "50": 127716137.0 + "95": 137065360.35 + "90": 135195515.68 + output_sequence_lengths: + total: 315398.0 + tpot: + total: 275332000.00000006 + system_tps: 162.2332 + system_desc.json: + tps_utilization: 0.024802 + results/system_i/deepseek-r1/r256: + accuracy_results.json: {} + config.yaml: + concurrency: 256 + point.yaml: + concurrency: 256 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 256 + requests_completed: 2560 + requests_issued: 2560 + result_summary.json: + n_samples_issued: 2759 + n_samples_completed: 2759 + duration_ns: 2418594571195.9995 + ttft: + total: 512378878272.0 + percentiles: + "50": 185711808.0 + "95": 842228387.0 + "90": 710925071.2 + output_sequence_lengths: + total: 10495599.0 + tpot: + total: 11171191000.000002 + system_tps: 4339.5446 + system_desc.json: + tps_utilization: 0.663424 + results/system_i/deepseek-r1/r32: + accuracy_results.json: {} + config.yaml: + concurrency: 32 + point.yaml: + concurrency: 32 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 32 + requests_completed: 320 + requests_issued: 320 + result_summary.json: + n_samples_issued: 516 + n_samples_completed: 516 + duration_ns: 2102160301502.9998 + ttft: + total: 72592828674.0 + percentiles: + "50": 140683776.5 + "95": 237608990.0 + "90": 218223947.3 + output_sequence_lengths: + total: 1986475.0 + tpot: + total: 2089284000.0000002 + system_tps: 944.9684 + system_desc.json: + tps_utilization: 0.144466 + results/system_i/deepseek-r1/r4: + accuracy_results.json: {} + config.yaml: + concurrency: 4 + point.yaml: + concurrency: 4 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 4 + requests_completed: 40 + requests_issued: 40 + result_summary.json: + n_samples_issued: 117 + n_samples_completed: 117 + duration_ns: 1962849595746.0 + ttft: + total: 14924097162.0 + percentiles: + "50": 127556386.0 + "95": 135825117.4 + "90": 134171371.12 + output_sequence_lengths: + total: 539431.0 + tpot: + total: 473733000.00000006 + system_tps: 274.8203 + system_desc.json: + tps_utilization: 0.042014 + results/system_i/deepseek-r1/r512: + accuracy_results.json: {} + config.yaml: + concurrency: 512 + point.yaml: + concurrency: 512 + offline: elected + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 512 + requests_completed: 5120 + requests_issued: 5120 + result_summary.json: + n_samples_issued: 4388 + n_samples_completed: 4388 + duration_ns: 2594198991881.0 + ttft: + total: 1184049108896.0 + percentiles: + "50": 269837992.0 + "95": 2367829187.25 + "90": 1948230948.2 + output_sequence_lengths: + total: 16968991.0 + tpot: + total: 17767012000.000004 + system_tps: 6541.1293 + system_desc.json: + tps_utilization: 1.0 + results/system_i/deepseek-r1/r64: + accuracy_results.json: {} + config.yaml: + concurrency: 64 + point.yaml: + concurrency: 64 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 64 + requests_completed: 640 + requests_issued: 640 + result_summary.json: + n_samples_issued: 906 + n_samples_completed: 906 + duration_ns: 2209290005442.0 + ttft: + total: 134375863380.0 + percentiles: + "50": 148317730.0 + "95": 300972722.5 + "90": 270441724.0 + output_sequence_lengths: + total: 3400893.0 + tpot: + total: 3668394000.0000005 + system_tps: 1539.3602 + system_desc.json: + tps_utilization: 0.235336 + results/system_i/deepseek-r1/r8: + accuracy_results.json: {} + config.yaml: + concurrency: 8 + point.yaml: + concurrency: 8 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 8 + requests_completed: 80 + requests_issued: 80 + result_summary.json: + n_samples_issued: 220 + n_samples_completed: 220 + duration_ns: 1919705564758.0 + ttft: + total: 28929776150.000004 + percentiles: + "50": 131498982.50000001 + "95": 139797791.85 + "90": 138138029.98 + output_sequence_lengths: + total: 867074.0 + tpot: + total: 890780000.0000001 + system_tps: 451.6703 + system_desc.json: + tps_utilization: 0.069051 diff --git a/tests/fixtures/validation/cases/sub_j.yaml b/tests/fixtures/validation/cases/sub_j.yaml new file mode 100644 index 000000000..87e422686 --- /dev/null +++ b/tests/fixtures/validation/cases/sub_j.yaml @@ -0,0 +1,480 @@ +defaults: + accuracy_results.json: + math500: + dataset_name: math500 + num_samples: 4388 + score: + exact_match: 81.3582 + extractor: BoxedAnswerExtractor + n_repeats: 1 + config.yaml: {} + point.yaml: + dataset: mlperf-perf-dataset-v1 + dataset_link: https://example.com/dataset + dataset_name: CNN/DailyMail + dataset_type: Performance + division: Standardized + link_to_model: https://example.com/model + link_to_model_transformation: https://example.com/quantization + max_supported_concurrency: 16384 + model_name: deepseek-r1 + model_notes: "" + model_precision: FP16 + runtime_settings: + load_pattern: concurrency + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true + seed_set: A + shared_docs: docs + shared_src: src + steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 + target_cohort: 2026-10-C1 + warmup: + data_source: mlperf-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + result_summary.json: + n_samples_failed: 0 + output_sequence_lengths: + percentiles: {} + tpot: + percentiles: + "50": 4000000.0 + "90": 5500000.0 + "95": 6000000.0 + "99": 6500000.0 + tps_per_user: 181.8182 + system_desc.json: + submitter_org_names: Submitter-J + submitter_contact: contact@example.com + system_name: system_j + system_category: datacenter + max_supported_concurrency: 16384 + serving_framework: TensorRT-LLM + node_types: + - system_node_ensemble_id: 0 + number_of_nodes: 1 + host_processor_model_name: NVIDIA Grace CPU (Armv9) + host_processors_per_node: 36 + host_processor_core_count: 72 + host_memory_capacity: 2.16 TB LPDDR5x + host_networking: InfiniBand NDR400 + host_storage_capacity: 30 TB + host_storage_type: NVMe SSD + operating_system: Ubuntu 22.04 (aarch64) + host_memory_configuration: 8x 64GB DDR5 + host_network_card_count: 1x NIC + driver: "1.0" + filesystem: ext4 + accelerator_info: + - accelerator_model_name: NVIDIA GB300 + accelerators_per_node: 72 + accelerator_memory_capacity: 288 GB HBM3e + accelerator_memory_type: HBM3e + accelerator_interconnect: NVLink 5.0 + division: Standardized + model_precision: FP16 + link_to_model: https://example.com/model + dataset_id: cnn_dailymail + dataset_name: CNN/DailyMail + input_token_average: 870.0 + output_token_average: 128.0 + dataset_type: performance + dataset_link: https://example.com/dataset + measured_accuracy_score: "38.7" + system_size: 1 node + model_id: deepseek-r1 + system_node_ensemble_count: 1 + system_node_ensemble_total: 1 + publication_status: Available + node_config: 8x H100 test node + config_summary: + disaggregated: null + expert_parallel: 1 + tensor_parallel: 72 + pipeline_parallel: 1 + data_parallel: 1 + batch: null + cooling: Liquid-cooled +files: + docs/.gitkeep: "" + results/system_j/system_power.json: + system_desc_id: system_j + cooling: liquid + node_sets: + - node_set_id: 0 + system_node_ensemble_id: 0 + nodes_provisioned: 1 + power_method: component_sum + components: + cpu: + model: NVIDIA Grace CPU (Armv9) + count_per_node: 36 + tdp_per_unit: + value_w: 350 + source_type: vendor_spec + source: https://example.com/cpu-spec + accelerator: + model: NVIDIA GB300 + count_per_node: 72 + tdp_per_unit: + value_w: 700 + source_type: vendor_spec + source: https://example.com/accelerator-spec + scale_up_network: + method: declared_tdp + switch_count: 1 + tdp_per_switch: + value_w: 3500 + source_type: vendor_spec + source: https://example.com/switch-spec + scale_out: + present: false + src/tensorrt-llm/README.md: "# tensorrt-llm + + + Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce + a point here. + + " +points: + results/system_j/deepseek-r1/r1024: + accuracy_results.json: {} + config.yaml: + concurrency: 1024 + point.yaml: + concurrency: 1024 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 1024 + requests_completed: 10240 + requests_issued: 10240 + result_summary.json: + n_samples_issued: 8776 + n_samples_completed: 8776 + duration_ns: 323142428984.0 + ttft: + total: 3105883110512.0 + percentiles: + "50": 353906462.0 + "95": 1762244150.0 + "90": 1480576612.4 + output_sequence_lengths: + total: 34315466.0 + tpot: + total: 35534024000.00001 + system_tps: 106193.0063 + system_desc.json: + tps_utilization: 0.283152 + results/system_j/deepseek-r1/r128: + accuracy_results.json: {} + config.yaml: + concurrency: 128 + point.yaml: + concurrency: 128 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 128 + requests_completed: 1280 + requests_issued: 1280 + result_summary.json: + n_samples_issued: 8776 + n_samples_completed: 8776 + duration_ns: 1335987189179.0002 + ttft: + total: 2152528849644.0 + percentiles: + "50": 245274481.5 + "95": 334071958.0 + "90": 316312462.7 + output_sequence_lengths: + total: 34063493.0 + tpot: + total: 35534024000.00001 + system_tps: 25496.871 + system_desc.json: + tps_utilization: 0.067985 + results/system_j/deepseek-r1/r16384: + accuracy_results.json: {} + config.yaml: + concurrency: 16384 + point.yaml: + concurrency: 16384 + offline: elected + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 16384 + requests_completed: 163840 + requests_issued: 163840 + result_summary.json: + n_samples_issued: 417284 + n_samples_completed: 417284 + duration_ns: 4165777685634.0 + ttft: + total: 178378755811144.0 + percentiles: + "50": 427475666.0 + "95": 8529496162.349946 + "90": 6909092063.079957 + output_sequence_lengths: + total: 1562330261.0 + tpot: + total: 1689582916000.0002 + system_tps: 375039.2793 + system_desc.json: + tps_utilization: 1.0 + results/system_j/deepseek-r1/r2048: + accuracy_results.json: {} + config.yaml: + concurrency: 2048 + point.yaml: + concurrency: 2048 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 2048 + requests_completed: 20480 + requests_issued: 20480 + result_summary.json: + n_samples_issued: 8776 + n_samples_completed: 8776 + duration_ns: 247915387632.0 + ttft: + total: 4262954461920.0 + percentiles: + "50": 485751420.0 + "95": 3795499436.5 + "90": 3133549833.2000003 + output_sequence_lengths: + total: 33975939.0 + tpot: + total: 35534024000.00001 + system_tps: 137046.5114 + system_desc.json: + tps_utilization: 0.365419 + results/system_j/deepseek-r1/r256: + accuracy_results.json: {} + config.yaml: + concurrency: 256 + point.yaml: + concurrency: 256 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 256 + requests_completed: 2560 + requests_issued: 2560 + result_summary.json: + n_samples_issued: 8776 + n_samples_completed: 8776 + duration_ns: 782307149059.0001 + ttft: + total: 2172863837720.0 + percentiles: + "50": 247591595.0 + "95": 505510358.75 + "90": 453926606.0 + output_sequence_lengths: + total: 33878035.0 + tpot: + total: 35534024000.00001 + system_tps: 43305.2862 + system_desc.json: + tps_utilization: 0.115469 + results/system_j/deepseek-r1/r32: + accuracy_results.json: {} + config.yaml: + concurrency: 32 + point.yaml: + concurrency: 32 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 32 + requests_completed: 320 + requests_issued: 320 + result_summary.json: + n_samples_issued: 8776 + n_samples_completed: 8776 + duration_ns: 5076457317773.0 + ttft: + total: 2109229364492.0 + percentiles: + "50": 240340629.5 + "95": 315733786.5 + "90": 300655155.1 + output_sequence_lengths: + total: 33926283.0 + tpot: + total: 35534024000.00001 + system_tps: 6683.0628 + system_desc.json: + tps_utilization: 0.01782 + results/system_j/deepseek-r1/r4096: + accuracy_results.json: {} + config.yaml: + concurrency: 4096 + point.yaml: + concurrency: 4096 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 4096 + requests_completed: 40960 + requests_issued: 40960 + result_summary.json: + n_samples_issued: 8776 + n_samples_completed: 8776 + duration_ns: 196988273307.0 + ttft: + total: 45268755452096.0 + percentiles: + "50": 5158244696.0 + "95": 6765218642.5 + "90": 6443823853.2 + output_sequence_lengths: + total: 34013009.0 + tpot: + total: 35534024000.00001 + system_tps: 172665.1461 + system_desc.json: + tps_utilization: 0.460392 + results/system_j/deepseek-r1/r512: + accuracy_results.json: {} + config.yaml: + concurrency: 512 + point.yaml: + concurrency: 512 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 512 + requests_completed: 5120 + requests_issued: 5120 + result_summary.json: + n_samples_issued: 8776 + n_samples_completed: 8776 + duration_ns: 495036806948.0 + ttft: + total: 2712508796676.0 + percentiles: + "50": 309082588.5 + "95": 569378711.0 + "90": 517319486.5 + output_sequence_lengths: + total: 34233018.0 + tpot: + total: 35534024000.00001 + system_tps: 69152.4701 + system_desc.json: + tps_utilization: 0.184387 + results/system_j/deepseek-r1/r64: + accuracy_results.json: {} + config.yaml: + concurrency: 64 + point.yaml: + concurrency: 64 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 64 + requests_completed: 640 + requests_issued: 640 + result_summary.json: + n_samples_issued: 8776 + n_samples_completed: 8776 + duration_ns: 2602571018770.0 + ttft: + total: 2027031584516.0 + percentiles: + "50": 230974428.5 + "95": 307411728.5 + "90": 292124268.5 + output_sequence_lengths: + total: 34185331.0 + tpot: + total: 35534024000.00001 + system_tps: 13135.2154 + system_desc.json: + tps_utilization: 0.035024 + results/system_j/deepseek-r1/r8192: + accuracy_results.json: {} + config.yaml: + concurrency: 8192 + point.yaml: + concurrency: 8192 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 8192 + requests_completed: 81920 + requests_issued: 81920 + result_summary.json: + n_samples_issued: 417284 + n_samples_completed: 417284 + duration_ns: 6231386322474.0 + ttft: + total: 165929879676176.0 + percentiles: + "50": 397642564.0 + "95": 641731587.85 + "90": 592913783.08 + output_sequence_lengths: + total: 1564302823.0 + tpot: + total: 1689582916000.0002 + system_tps: 251036.0844 + system_desc.json: + tps_utilization: 0.669359 diff --git a/tests/fixtures/validation/cases/valid_standardized.yaml b/tests/fixtures/validation/cases/valid_standardized.yaml new file mode 100644 index 000000000..88e01ab88 --- /dev/null +++ b/tests/fixtures/validation/cases/valid_standardized.yaml @@ -0,0 +1,405 @@ +defaults: + accuracy_results.json: + llm-perf-dataset-v1: + dataset_name: llm-perf-dataset-v1 + num_samples: 13368 + extractor: RougeExtractor + score: + rouge1: "45.12" + rouge2: "22.01" + rougeL: "30.45" + n_repeats: 1 + config.yaml: {} + point.yaml: + dataset: llm-perf-dataset-v1 + dataset_link: https://example.com/dataset + dataset_name: CNN/DailyMail + dataset_type: Performance + division: Standardized + link_to_model: https://example.com/model + link_to_model_transformation: https://example.com/quantization + max_supported_concurrency: 1000 + model_name: llama3_1-8b + model_notes: "" + model_precision: FP16 + runtime_settings: + load_pattern: concurrency + runtime: + model_seed: 9315206023656308754 + sample_index_rng_seed: 586478644936801402 + scheduler_rng_seed: 10487924139932647040 + stream_all_chunks: true + seed_set: A + shared_docs: docs + shared_src: src + steady_state: + state: + tpot_p50: Plateau + tpot_p90: Plateau + ttft_p50: Plateau + ttft_p90: Plateau + status: windowable + verdict: STEADY STATE + window: + n_samples: 4000 + super_pass_end: 4 + super_pass_size: 1000 + super_pass_start: 1 + target_cohort: 2026-10-C1 + warmup: + data_source: llm-perf-dataset-v1 validation split + duration_s: 60.0 + initialization_steps: + - model loaded + - kv-cache warmed + link_logs: https://example.com/warmup-logs + logs_retained: true + result_summary.json: + n_samples_failed: 0 + output_sequence_lengths: + percentiles: {} + tpot: + percentiles: + "50": 4000000.0 + "90": 5500000.0 + "95": 6000000.0 + "99": 6500000.0 + tps_per_user: 181.8182 + system_desc.json: + submitter_org_names: ACME Corp + submitter_contact: contact@example.com + system_name: ACME H100x8 Node + system_category: datacenter + max_supported_concurrency: 1000 + serving_framework: vLLM 0.4.2 + node_types: + - system_node_ensemble_id: 0 + number_of_nodes: 1 + host_processor_model_name: AMD EPYC 9654 + host_processors_per_node: 2 + host_processor_core_count: 96 + host_memory_capacity: 768 GB + host_networking: InfiniBand HDR + host_storage_capacity: 20 TB + host_storage_type: NVMe SSD + cooling: Air-cooled + operating_system: Ubuntu 22.04 + host_memory_configuration: 8x 64GB DDR5 + host_network_card_count: 1x NIC + driver: "1.0" + filesystem: ext4 + accelerator_info: + - accelerator_model_name: NVIDIA H100 SXM5 80GB + accelerators_per_node: 8 + accelerator_memory_capacity: 80 GB HBM3 + accelerator_memory_type: HBM3e + accelerator_interconnect: NVLink 4.0 900 GB/s + accelerator_host_interconnect: PCIe Gen5 + division: Standardized + model_precision: FP16 + link_to_model: https://example.com/model + dataset_id: cnn_dailymail + dataset_name: CNN/DailyMail + input_token_average: 870.0 + output_token_average: 128.0 + dataset_type: performance + dataset_link: https://example.com/dataset + measured_accuracy_score: "38.7" + system_size: 1 node + model_id: llama3.1-8b + system_node_ensemble_count: 1 + system_node_ensemble_total: 1 + model_name: llama3.1-8b + publication_status: Available + node_config: 8x H100 test node + config_summary: + disaggregated: null + expert_parallel: 1 + tensor_parallel: 8 + pipeline_parallel: 1 + data_parallel: 1 + batch: null +files: + docs/.gitkeep: "" + docs/calibration.adoc: "= Weight Transformation Documentation + + + == Quantization + + + Applied INT8 post-training quantization using GPTQ with 512-sample calibration dataset drawn from + the performance dataset. No gradient updates applied. Calibration dataset disclosed in full in calibration_data/. + + " + results/system_k/system_power.json: + system_desc_id: system_k + cooling: air + node_sets: + - node_set_id: 0 + system_node_ensemble_id: 0 + nodes_provisioned: 1 + power_method: component_sum + components: + cpu: + model: AMD EPYC 9654 + count_per_node: 2 + tdp_per_unit: + value_w: 350 + source_type: vendor_spec + source: https://example.com/cpu-spec + accelerator: + model: NVIDIA H100 SXM5 80GB + count_per_node: 8 + tdp_per_unit: + value_w: 700 + source_type: vendor_spec + source: https://example.com/accelerator-spec + scale_up_network: + method: declared_tdp + switch_count: 1 + tdp_per_switch: + value_w: 3500 + source_type: vendor_spec + source: https://example.com/switch-spec + scale_out: + present: false + src/llama3_1-8b/.gitkeep: "" + src/llama3_1-8b/README.md: "# llama3_1-8b + + + How to build the SUT and reproduce a Pareto point. + + " + src/vllm/README.md: "# vllm + + + Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce + a point here. + + " +points: + results/system_k/llama3_1-8b/r1000: + accuracy_results.json: {} + config.yaml: + concurrency: 1000 + point.yaml: + concurrency: 1000 + offline: elected + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 1000 + requests_completed: 10000 + requests_issued: 10000 + result_summary.json: + n_samples_issued: 22000 + n_samples_completed: 22000 + duration_ns: 1200000000000 + ttft: + total: 9259800000000.0 + percentiles: + "50": 420900000.0 + "95": 890500000.0 + "90": 796580000.0 + output_sequence_lengths: + total: 5400000.0 + tpot: + total: 89078000000.00002 + system_tps: 4500.0 + system_desc.json: + tps_utilization: 0.978261 + results/system_k/llama3_1-8b/r16: + accuracy_results.json: {} + config.yaml: + concurrency: 16 + point.yaml: + concurrency: 16 + runtime_settings: + min_duration_ms: 600000 + steady_state: + window: + duration_s: 600.0 + warmup: + concurrency: 16 + requests_completed: 160 + requests_issued: 160 + result_summary.json: + n_samples_issued: 3000 + n_samples_completed: 3000 + duration_ns: 600000000000 + ttft: + total: 135600000000.0 + percentiles: + "50": 45200000.0 + "95": 89100000.0 + "90": 80320000.0 + output_sequence_lengths: + total: 480000.0 + tpot: + total: 12147000000.000002 + system_tps: 800.0 + system_desc.json: + tps_utilization: 0.173913 + results/system_k/llama3_1-8b/r20: + accuracy_results.json: {} + config.yaml: + concurrency: 38 + point.yaml: + concurrency: 20 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 20 + requests_completed: 200 + requests_issued: 200 + result_summary.json: + n_samples_issued: 8000 + n_samples_completed: 8000 + duration_ns: 1200000000000 + ttft: + total: 418400000000.0 + percentiles: + "50": 52300000.0 + "95": 110400000.0 + "90": 98780000.0 + output_sequence_lengths: + total: 1800000.0 + tpot: + total: 32392000000.000004 + system_tps: 1500.0 + system_desc.json: + tps_utilization: 0.326087 + results/system_k/llama3_1-8b/r256: + accuracy_results.json: {} + config.yaml: + concurrency: 256 + point.yaml: + concurrency: 256 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 256 + requests_completed: 2560 + requests_issued: 2560 + result_summary.json: + n_samples_issued: 20000 + n_samples_completed: 20000 + duration_ns: 1200000000000 + ttft: + total: 2050000000000.0 + percentiles: + "50": 102500000.0 + "95": 220800000.0 + "90": 197140000.0 + output_sequence_lengths: + total: 4800000.0 + tpot: + total: 80980000000.00002 + system_tps: 4000.0 + system_desc.json: + tps_utilization: 0.869565 + results/system_k/llama3_1-8b/r512: + accuracy_results.json: {} + config.yaml: + concurrency: 512 + point.yaml: + concurrency: 512 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 512 + requests_completed: 5120 + requests_issued: 5120 + result_summary.json: + n_samples_issued: 22000 + n_samples_completed: 22000 + duration_ns: 1200000000000 + ttft: + total: 3966600000000.0 + percentiles: + "50": 180300000.0 + "95": 410200000.0 + "90": 364220000.0 + output_sequence_lengths: + total: 5400000.0 + tpot: + total: 89078000000.00002 + system_tps: 4500.0 + system_desc.json: + tps_utilization: 0.978261 + results/system_k/llama3_1-8b/r768: + accuracy_results.json: {} + config.yaml: + concurrency: 768 + point.yaml: + concurrency: 768 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 768 + requests_completed: 7680 + requests_issued: 7680 + result_summary.json: + n_samples_issued: 23000 + n_samples_completed: 23000 + duration_ns: 1200000000000 + ttft: + total: 6801100000000.0 + percentiles: + "50": 295700000.0 + "95": 620100000.0 + "90": 555220000.0 + output_sequence_lengths: + total: 5520000.0 + tpot: + total: 93127000000.00002 + system_tps: 4600.0 + system_desc.json: + tps_utilization: 1.0 + results/system_k/llama3_1-8b/r88: + accuracy_results.json: {} + config.yaml: + concurrency: 88 + point.yaml: + concurrency: 88 + runtime_settings: + min_duration_ms: 1200000 + steady_state: + window: + duration_s: 1200.0 + warmup: + concurrency: 88 + requests_completed: 880 + requests_issued: 880 + result_summary.json: + n_samples_issued: 12000 + n_samples_completed: 12000 + duration_ns: 1200000000000 + ttft: + total: 817200000000.0 + percentiles: + "50": 68100000.0 + "95": 145200000.0 + "90": 129780000.0 + output_sequence_lengths: + total: 3000000.0 + tpot: + total: 48588000000.00001 + system_tps: 2500.0 + system_desc.json: + tps_utilization: 0.543478 diff --git a/tests/fixtures/validation/submissions/invalid_submission/docs/.gitkeep b/tests/fixtures/validation/submissions/invalid_submission/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/accuracy_results.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/accuracy_results.json deleted file mode 100644 index 49247cf54..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/accuracy_results.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "cnn_dailymail::llama3_8b": { - "dataset_name": "cnn_dailymail::llama3_8b", - "num_samples": 500, - "extractor": "RougeExtractor", - "score": { - "rouge1": "30.00", - "rouge2": "12.00", - "rougeL": "20.00" - }, - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/config.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/config.yaml deleted file mode 100644 index f9afe658a..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 16 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/point.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/point.yaml deleted file mode 100644 index cdfb6a1db..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 16 -dataset: llm-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 88 -model_name: Llama-3_1-8B-Instruct -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 16 - data_source: llm-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 160 - requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/result_summary.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/result_summary.json deleted file mode 100644 index e6e2bc2a1..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 3000, - "n_samples_completed": 3000, - "n_samples_failed": 0, - "duration_ns": 600000000000, - "ttft": { - "total": 135600000000.0, - "percentiles": { - "50": 45200000.0, - "95": 89100000.0, - "90": 80320000.0 - } - }, - "output_sequence_lengths": { - "total": 480000.0, - "percentiles": {} - }, - "tpot": { - "total": 12147000000.000002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 800.0, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/system_desc.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/system_desc.json deleted file mode 100644 index 4345d8064..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r16/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Invalid Corp", - "submitter_contact": "contact@example.com", - "system_name": "system_l", - "system_category": "datacenter", - "max_supported_concurrency": 88, - "serving_framework": "custom 0.0.1", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Generic CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 8, - "host_memory_capacity": "64 GB", - "host_networking": "Ethernet 1Gb", - "host_storage_capacity": "1 TB", - "host_storage_type": "HDD", - "operating_system": "Ubuntu 20.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Generic GPU", - "accelerators_per_node": 1, - "accelerator_memory_capacity": "16 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "Llama-3_1-8B-Instruct", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 1, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.32 -} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/accuracy_results.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/accuracy_results.json deleted file mode 100644 index 49247cf54..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/accuracy_results.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "cnn_dailymail::llama3_8b": { - "dataset_name": "cnn_dailymail::llama3_8b", - "num_samples": 500, - "extractor": "RougeExtractor", - "score": { - "rouge1": "30.00", - "rouge2": "12.00", - "rougeL": "20.00" - }, - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/config.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/config.yaml deleted file mode 100644 index 1a6f2cb8e..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 38 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/point.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/point.yaml deleted file mode 100644 index 821700459..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 38 -dataset: llm-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 88 -model_name: Llama-3_1-8B-Instruct -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 38 - data_source: llm-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 380 - requests_issued: 380 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/result_summary.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/result_summary.json deleted file mode 100644 index 0fe18b542..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 8000, - "n_samples_completed": 8000, - "n_samples_failed": 0, - "duration_ns": 1200000000000, - "ttft": { - "total": 361600000000.0, - "percentiles": { - "50": 45200000.0, - "95": 89100000.0, - "90": 80320000.0 - } - }, - "output_sequence_lengths": { - "total": 1800000.0, - "percentiles": {} - }, - "tpot": { - "total": 32392000000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1500.0, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/system_desc.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/system_desc.json deleted file mode 100644 index 6ee2cc387..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r38/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Invalid Corp", - "submitter_contact": "contact@example.com", - "system_name": "system_l", - "system_category": "datacenter", - "max_supported_concurrency": 88, - "serving_framework": "custom 0.0.1", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Generic CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 8, - "host_memory_capacity": "64 GB", - "host_networking": "Ethernet 1Gb", - "host_storage_capacity": "1 TB", - "host_storage_type": "HDD", - "operating_system": "Ubuntu 20.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Generic GPU", - "accelerators_per_node": 1, - "accelerator_memory_capacity": "16 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "Llama-3_1-8B-Instruct", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 1, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.6 -} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/accuracy_results.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/accuracy_results.json deleted file mode 100644 index 49247cf54..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/accuracy_results.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "cnn_dailymail::llama3_8b": { - "dataset_name": "cnn_dailymail::llama3_8b", - "num_samples": 500, - "extractor": "RougeExtractor", - "score": { - "rouge1": "30.00", - "rouge2": "12.00", - "rougeL": "20.00" - }, - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/config.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/config.yaml deleted file mode 100644 index cf16abd9b..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 88 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/point.yaml b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/point.yaml deleted file mode 100644 index 42a5783f8..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/point.yaml +++ /dev/null @@ -1,50 +0,0 @@ -concurrency: 88 -dataset: llm-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 88 -model_name: Llama-3_1-8B-Instruct -model_notes: "" -model_precision: FP16 -offline: elected -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 88 - data_source: llm-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 880 - requests_issued: 880 diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/result_summary.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/result_summary.json deleted file mode 100644 index 839c42f09..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 5000, - "n_samples_completed": 5000, - "n_samples_failed": 0, - "duration_ns": 1200000000000, - "ttft": { - "total": 226000000000.0, - "percentiles": { - "50": 45200000.0, - "95": 89100000.0, - "90": 80320000.0 - } - }, - "output_sequence_lengths": { - "total": 3000000.0, - "percentiles": {} - }, - "tpot": { - "total": 20245000000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 2500.0, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/system_desc.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/system_desc.json deleted file mode 100644 index 5210d5257..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/Llama-3_1-8B-Instruct/r88/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Invalid Corp", - "submitter_contact": "contact@example.com", - "system_name": "system_l", - "system_category": "datacenter", - "max_supported_concurrency": 88, - "serving_framework": "custom 0.0.1", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Generic CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 8, - "host_memory_capacity": "64 GB", - "host_networking": "Ethernet 1Gb", - "host_storage_capacity": "1 TB", - "host_storage_type": "HDD", - "operating_system": "Ubuntu 20.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Generic GPU", - "accelerators_per_node": 1, - "accelerator_memory_capacity": "16 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "Llama-3_1-8B-Instruct", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 1, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 1.0 -} diff --git a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/system_power.json b/tests/fixtures/validation/submissions/invalid_submission/results/system_l/system_power.json deleted file mode 100644 index 4fa3752e7..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/results/system_l/system_power.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "system_desc_id": "system_l", - "cooling": "air", - "node_sets": [ - { - "node_set_id": 0, - "system_node_ensemble_id": 0, - "nodes_provisioned": 1, - "power_method": "component_sum", - "components": { - "cpu": { - "model": "Generic CPU", - "count_per_node": 1, - "tdp_per_unit": { - "value_w": 350, - "source_type": "vendor_spec", - "source": "https://example.com/cpu-spec" - } - }, - "accelerator": { - "model": "Generic GPU", - "count_per_node": 1, - "tdp_per_unit": { - "value_w": 700, - "source_type": "vendor_spec", - "source": "https://example.com/accelerator-spec" - } - }, - "scale_up_network": { - "method": "declared_tdp", - "switch_count": 1, - "tdp_per_switch": { - "value_w": 3500, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - } - } - ], - "scale_out": { - "present": false - } -} diff --git a/tests/fixtures/validation/submissions/invalid_submission/src/custom/README.md b/tests/fixtures/validation/submissions/invalid_submission/src/custom/README.md deleted file mode 100644 index 25fbd778e..000000000 --- a/tests/fixtures/validation/submissions/invalid_submission/src/custom/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# custom - -Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_a/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_a/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/config.yaml deleted file mode 100644 index b8581faa1..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/point.yaml deleted file mode 100644 index 092fd8959..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 1024 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1024 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 10240 - requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/result_summary.json deleted file mode 100644 index 07c4aa73c..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 2302757311892.0, - "ttft": { - "total": 551183759784528.0, - "percentiles": { - "50": 86176322668.0, - "95": 100733262106.25, - "90": 97821874218.6 - } - }, - "output_sequence_lengths": { - "total": 22417180.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 9734.9295, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/system_desc.json deleted file mode 100644 index 3e06b504a..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r1024/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-A", - "submitter_contact": "contact@example.com", - "system_name": "system_a", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.997637 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/config.yaml deleted file mode 100644 index 51457c3cf..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/point.yaml deleted file mode 100644 index 8c27c4dd8..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 128 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 128 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 1280 - requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/result_summary.json deleted file mode 100644 index d04481dea..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 3940004920032.0, - "ttft": { - "total": 1084206049836.0, - "percentiles": { - "50": 169513141.0, - "95": 261888051.50000003, - "90": 243413069.40000004 - } - }, - "output_sequence_lengths": { - "total": 22407098.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 5687.0736, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/system_desc.json deleted file mode 100644 index 81ae74864..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r128/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-A", - "submitter_contact": "contact@example.com", - "system_name": "system_a", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.582812 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/config.yaml deleted file mode 100644 index f9afe658a..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/point.yaml deleted file mode 100644 index 7864405d8..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 16 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 16 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 160 - requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/result_summary.json deleted file mode 100644 index 41eff27bd..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 11856610466381.0, - "ttft": { - "total": 599489491146.0, - "percentiles": { - "50": 93728813.5, - "95": 145818384.25, - "90": 135400470.1 - } - }, - "output_sequence_lengths": { - "total": 22387661.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1888.2008, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/system_desc.json deleted file mode 100644 index 92c85aa89..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r16/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-A", - "submitter_contact": "contact@example.com", - "system_name": "system_a", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.193503 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/config.yaml deleted file mode 100644 index 6b53d1bba..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 2048 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/point.yaml deleted file mode 100644 index 29199e39a..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/point.yaml +++ /dev/null @@ -1,50 +0,0 @@ -concurrency: 2048 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -offline: elected -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 2048 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 20480 - requests_issued: 20480 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/result_summary.json deleted file mode 100644 index db91311d5..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 2294246323608.9995, - "ttft": { - "total": 2845856057894070.0, - "percentiles": { - "50": 444943098482.5, - "95": 462411343300.0, - "90": 458917694336.5 - } - }, - "output_sequence_lengths": { - "total": 22387231.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 9757.9893, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/system_desc.json deleted file mode 100644 index 5d6067749..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r2048/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-A", - "submitter_contact": "contact@example.com", - "system_name": "system_a", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 1.0 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/config.yaml deleted file mode 100644 index 24d4dedbd..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 4 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/point.yaml deleted file mode 100644 index 1a6d3dfe2..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 4 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 4 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 40 - requests_issued: 40 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/result_summary.json deleted file mode 100644 index 32bec22e7..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 28913949735508.0, - "ttft": { - "total": 521833969800.0, - "percentiles": { - "50": 81587550.0, - "95": 183622220.0, - "90": 163215286.0 - } - }, - "output_sequence_lengths": { - "total": 22427551.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 775.6654, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/system_desc.json deleted file mode 100644 index c496c2aac..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r4/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-A", - "submitter_contact": "contact@example.com", - "system_name": "system_a", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.07949 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/config.yaml deleted file mode 100644 index 846832e5e..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/point.yaml deleted file mode 100644 index a3b2de6dd..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 512 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 512 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 5120 - requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/result_summary.json deleted file mode 100644 index 2fb9226cc..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 2427693763211.0, - "ttft": { - "total": 2446935638394.0, - "percentiles": { - "50": 382572801.5, - "95": 10778207187.0, - "90": 8699080309.900002 - } - }, - "output_sequence_lengths": { - "total": 22315593.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 9192.0955, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/system_desc.json deleted file mode 100644 index db77d10fc..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r512/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-A", - "submitter_contact": "contact@example.com", - "system_name": "system_a", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.942007 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/config.yaml deleted file mode 100644 index 7db6216f5..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/point.yaml deleted file mode 100644 index c57905824..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 64 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 64 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 640 - requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/result_summary.json deleted file mode 100644 index f69318dbf..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 5163749763587.001, - "ttft": { - "total": 814271667534.0, - "percentiles": { - "50": 127309516.5, - "95": 193387620.0, - "90": 180171999.3 - } - }, - "output_sequence_lengths": { - "total": 22246057.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 4308.1207, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/system_desc.json deleted file mode 100644 index 41ffa1e8e..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/gpt-oss-120b/r64/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-A", - "submitter_contact": "contact@example.com", - "system_name": "system_a", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.441497 -} diff --git a/tests/fixtures/validation/submissions/sub_a/results/system_a/system_power.json b/tests/fixtures/validation/submissions/sub_a/results/system_a/system_power.json deleted file mode 100644 index 3109dbf5f..000000000 --- a/tests/fixtures/validation/submissions/sub_a/results/system_a/system_power.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "system_desc_id": "system_a", - "cooling": "air", - "node_sets": [ - { - "node_set_id": 0, - "system_node_ensemble_id": 0, - "nodes_provisioned": 1, - "power_method": "component_sum", - "components": { - "cpu": { - "model": "AMD EPYC 9654", - "count_per_node": 2, - "tdp_per_unit": { - "value_w": 350, - "source_type": "vendor_spec", - "source": "https://example.com/cpu-spec" - } - }, - "accelerator": { - "model": "AMD Instinct MI355X", - "count_per_node": 8, - "tdp_per_unit": { - "value_w": 700, - "source_type": "vendor_spec", - "source": "https://example.com/accelerator-spec" - } - }, - "scale_up_network": { - "method": "declared_tdp", - "switch_count": 1, - "tdp_per_switch": { - "value_w": 3500, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - } - } - ], - "scale_out": { - "present": false - } -} diff --git a/tests/fixtures/validation/submissions/sub_a/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_a/src/vllm/README.md deleted file mode 100644 index be3edfb30..000000000 --- a/tests/fixtures/validation/submissions/sub_a/src/vllm/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# vllm - -Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_b/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_b/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/config.yaml deleted file mode 100644 index b8581faa1..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/point.yaml deleted file mode 100644 index 092fd8959..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 1024 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1024 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 10240 - requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/result_summary.json deleted file mode 100644 index ef0d7d52f..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 2555878737455.0, - "ttft": { - "total": 607712738731644.0, - "percentiles": { - "50": 95014499489.0, - "95": 113055413992.5, - "90": 109447231091.8 - } - }, - "output_sequence_lengths": { - "total": 22287872.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 8720.2384, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/system_desc.json deleted file mode 100644 index f8099d87e..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r1024/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-B", - "submitter_contact": "contact@example.com", - "system_name": "system_b", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 16, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 16, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 1.0 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/config.yaml deleted file mode 100644 index 51457c3cf..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/point.yaml deleted file mode 100644 index 8c27c4dd8..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 128 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 128 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 1280 - requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/result_summary.json deleted file mode 100644 index c10338bc1..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 4564665788130.0, - "ttft": { - "total": 1276608945222.0, - "percentiles": { - "50": 199594894.5, - "95": 306682164.0, - "90": 285264710.1 - } - }, - "output_sequence_lengths": { - "total": 22291611.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 4883.5144, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/system_desc.json deleted file mode 100644 index 4de583965..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r128/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-B", - "submitter_contact": "contact@example.com", - "system_name": "system_b", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 16, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 16, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.560021 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/config.yaml deleted file mode 100644 index f9afe658a..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/point.yaml deleted file mode 100644 index 7864405d8..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 16 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 16 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 160 - requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/result_summary.json deleted file mode 100644 index e6d7a27c2..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 13160683481153.0, - "ttft": { - "total": 707502001908.0, - "percentiles": { - "50": 110616323.0, - "95": 220638667.75, - "90": 198634198.8 - } - }, - "output_sequence_lengths": { - "total": 22403343.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1702.2933, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/system_desc.json deleted file mode 100644 index fefbabe16..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r16/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-B", - "submitter_contact": "contact@example.com", - "system_name": "system_b", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 16, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 16, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.195212 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/config.yaml deleted file mode 100644 index 6b53d1bba..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 2048 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/point.yaml deleted file mode 100644 index 29199e39a..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/point.yaml +++ /dev/null @@ -1,50 +0,0 @@ -concurrency: 2048 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -offline: elected -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 2048 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 20480 - requests_issued: 20480 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/result_summary.json deleted file mode 100644 index 7db6a29de..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 2587004792635.0, - "ttft": { - "total": 3198282303472386.0, - "percentiles": { - "50": 500044137503.5, - "95": 545254932484.0, - "90": 536212773487.9 - } - }, - "output_sequence_lengths": { - "total": 22359894.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 8643.1591, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/system_desc.json deleted file mode 100644 index fa81d1590..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r2048/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-B", - "submitter_contact": "contact@example.com", - "system_name": "system_b", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 16, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 16, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.991161 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/config.yaml deleted file mode 100644 index 24d4dedbd..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 4 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/point.yaml deleted file mode 100644 index 1a6d3dfe2..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 4 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 4 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 40 - requests_issued: 40 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/result_summary.json deleted file mode 100644 index a8be486a4..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 30408127168980.996, - "ttft": { - "total": 562383145116.0, - "percentiles": { - "50": 87927321.0, - "95": 197007595.0, - "90": 175191540.2 - } - }, - "output_sequence_lengths": { - "total": 22330923.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 734.3735, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/system_desc.json deleted file mode 100644 index 66eeeb0f6..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r4/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-B", - "submitter_contact": "contact@example.com", - "system_name": "system_b", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 16, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 16, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.084215 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/config.yaml deleted file mode 100644 index 846832e5e..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/point.yaml deleted file mode 100644 index a3b2de6dd..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 512 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 512 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 5120 - requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/result_summary.json deleted file mode 100644 index 62292160c..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 2755067877868.0, - "ttft": { - "total": 2813058879462.0, - "percentiles": { - "50": 439815334.5, - "95": 11423621862.5, - "90": 9226860556.9 - } - }, - "output_sequence_lengths": { - "total": 22400639.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 8130.7031, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/system_desc.json deleted file mode 100644 index c1b9445fc..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r512/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-B", - "submitter_contact": "contact@example.com", - "system_name": "system_b", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 16, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 16, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.932395 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/accuracy_results.json deleted file mode 100644 index b59ac0f79..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 4395, - "score": { - "exact_match": 83.13 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/config.yaml deleted file mode 100644 index 7db6216f5..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/point.yaml deleted file mode 100644 index c57905824..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 64 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: gpt-oss-120b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 64 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 640 - requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/result_summary.json deleted file mode 100644 index c26fef001..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 6396, - "n_samples_completed": 6396, - "n_samples_failed": 0, - "duration_ns": 6369959177476.999, - "ttft": { - "total": 994203277548.0, - "percentiles": { - "50": 155441413.0, - "95": 237329605.75, - "90": 220951967.2 - } - }, - "output_sequence_lengths": { - "total": 22395866.0, - "percentiles": {} - }, - "tpot": { - "total": 25897404000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 3515.8571, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/system_desc.json deleted file mode 100644 index adcc5e3b3..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/gpt-oss-120b/r64/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-B", - "submitter_contact": "contact@example.com", - "system_name": "system_b", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "AMD Instinct MI355X", - "accelerators_per_node": 16, - "accelerator_memory_capacity": "288 GB", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "gpt-oss-120b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 16, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.403184 -} diff --git a/tests/fixtures/validation/submissions/sub_b/results/system_b/system_power.json b/tests/fixtures/validation/submissions/sub_b/results/system_b/system_power.json deleted file mode 100644 index 0db242472..000000000 --- a/tests/fixtures/validation/submissions/sub_b/results/system_b/system_power.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "system_desc_id": "system_b", - "cooling": "air", - "node_sets": [ - { - "node_set_id": 0, - "system_node_ensemble_id": 0, - "nodes_provisioned": 1, - "power_method": "component_sum", - "components": { - "cpu": { - "model": "AMD EPYC 9654", - "count_per_node": 2, - "tdp_per_unit": { - "value_w": 350, - "source_type": "vendor_spec", - "source": "https://example.com/cpu-spec" - } - }, - "accelerator": { - "model": "AMD Instinct MI355X", - "count_per_node": 16, - "tdp_per_unit": { - "value_w": 700, - "source_type": "vendor_spec", - "source": "https://example.com/accelerator-spec" - } - }, - "scale_up_network": { - "method": "declared_tdp", - "switch_count": 1, - "tdp_per_switch": { - "value_w": 3500, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - } - } - ], - "scale_out": { - "present": false - } -} diff --git a/tests/fixtures/validation/submissions/sub_b/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_b/src/vllm/README.md deleted file mode 100644 index be3edfb30..000000000 --- a/tests/fixtures/validation/submissions/sub_b/src/vllm/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# vllm - -Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_c/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_c/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/config.yaml deleted file mode 100644 index 51457c3cf..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/point.yaml deleted file mode 100644 index cd5f4fc95..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 128 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 128 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 1280 - requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/result_summary.json deleted file mode 100644 index fc8a189c0..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 3704248419955.0, - "ttft": { - "total": 5674433556480.0, - "percentiles": { - "50": 230893292.5, - "95": 330418109.5, - "90": 310513146.1 - } - }, - "output_sequence_lengths": { - "total": 7189576.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1940.9001, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/system_desc.json deleted file mode 100644 index 702c56e45..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r128/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-C", - "submitter_contact": "contact@example.com", - "system_name": "system_c", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 4, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 4, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.686722 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/config.yaml deleted file mode 100644 index f9afe658a..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/point.yaml deleted file mode 100644 index 49f86ecbf..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 16 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 16 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 160 - requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/result_summary.json deleted file mode 100644 index 13e0192b4..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 9681657879146.0, - "ttft": { - "total": 2364827467776.0, - "percentiles": { - "50": 96225076.0, - "95": 164579607.0, - "90": 150908700.8 - } - }, - "output_sequence_lengths": { - "total": 7193577.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 743.0109, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/system_desc.json deleted file mode 100644 index 89c194f23..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r16/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-C", - "submitter_contact": "contact@example.com", - "system_name": "system_c", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 4, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 4, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.262889 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/config.yaml deleted file mode 100644 index f928653b5..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/point.yaml deleted file mode 100644 index 71e2cd93e..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 256 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 256 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 2560 - requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/result_summary.json deleted file mode 100644 index e49af2020..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 3055408461497.0005, - "ttft": { - "total": 8782727024640.0, - "percentiles": { - "50": 357370077.5, - "95": 457645674.25, - "90": 437590554.9 - } - }, - "output_sequence_lengths": { - "total": 7178996.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 2349.6027, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/system_desc.json deleted file mode 100644 index afc25595f..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r256/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-C", - "submitter_contact": "contact@example.com", - "system_name": "system_c", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 4, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 4, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.831327 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/config.yaml deleted file mode 100644 index b03f20973..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 32 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/point.yaml deleted file mode 100644 index 81faebf4f..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 32 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 32 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 320 - requests_issued: 320 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/result_summary.json deleted file mode 100644 index a53ebf6c8..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 7215663280145.0, - "ttft": { - "total": 3557367164928.0, - "percentiles": { - "50": 144749640.5, - "95": 211749072.25, - "90": 198349185.9 - } - }, - "output_sequence_lengths": { - "total": 7193551.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 996.9355, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/system_desc.json deleted file mode 100644 index d4d3e195c..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r32/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-C", - "submitter_contact": "contact@example.com", - "system_name": "system_c", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 4, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 4, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.352732 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/config.yaml deleted file mode 100644 index 846832e5e..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/point.yaml deleted file mode 100644 index bc041691d..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/point.yaml +++ /dev/null @@ -1,50 +0,0 @@ -concurrency: 512 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -offline: elected -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 512 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 5120 - requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/result_summary.json deleted file mode 100644 index d3f1c582f..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 2550450936890.0, - "ttft": { - "total": 13680033595392.0, - "percentiles": { - "50": 556641992.0, - "95": 841800473.5, - "90": 784768777.2 - } - }, - "output_sequence_lengths": { - "total": 7208408.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 2826.3269, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/system_desc.json deleted file mode 100644 index f0ae6e6b7..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r512/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-C", - "submitter_contact": "contact@example.com", - "system_name": "system_c", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 4, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 4, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 1.0 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/config.yaml deleted file mode 100644 index 7db6216f5..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/point.yaml deleted file mode 100644 index f7a332096..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 64 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 64 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 640 - requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/result_summary.json deleted file mode 100644 index d7c5651f6..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 4947046368648.0, - "ttft": { - "total": 3985956200448.0, - "percentiles": { - "50": 162188973.0, - "95": 255115666.5, - "90": 236530327.8 - } - }, - "output_sequence_lengths": { - "total": 7194216.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1454.2447, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/system_desc.json deleted file mode 100644 index 16837b117..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r64/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-C", - "submitter_contact": "contact@example.com", - "system_name": "system_c", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 4, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 4, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.514535 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/config.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/config.yaml deleted file mode 100644 index e1b56a7e8..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 8 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/point.yaml b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/point.yaml deleted file mode 100644 index f195dbf92..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 8 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 8 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 80 - requests_issued: 80 diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/result_summary.json deleted file mode 100644 index 35c0965bb..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 17117165266230.002, - "ttft": { - "total": 2276072361984.0, - "percentiles": { - "50": 92613621.5, - "95": 157459882.0, - "90": 144490629.9 - } - }, - "output_sequence_lengths": { - "total": 7185824.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 419.8022, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/system_desc.json deleted file mode 100644 index a25fa6d35..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/qwen3-coder-480b/r8/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-C", - "submitter_contact": "contact@example.com", - "system_name": "system_c", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 4, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 4, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.148533 -} diff --git a/tests/fixtures/validation/submissions/sub_c/results/system_c/system_power.json b/tests/fixtures/validation/submissions/sub_c/results/system_c/system_power.json deleted file mode 100644 index ba1355173..000000000 --- a/tests/fixtures/validation/submissions/sub_c/results/system_c/system_power.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "system_desc_id": "system_c", - "cooling": "liquid", - "node_sets": [ - { - "node_set_id": 0, - "system_node_ensemble_id": 0, - "nodes_provisioned": 1, - "power_method": "component_sum", - "components": { - "cpu": { - "model": "Custom TPU Host CPU", - "count_per_node": 1, - "tdp_per_unit": { - "value_w": 350, - "source_type": "vendor_spec", - "source": "https://example.com/cpu-spec" - } - }, - "accelerator": { - "model": "Google TPU v5p", - "count_per_node": 4, - "tdp_per_unit": { - "value_w": 700, - "source_type": "vendor_spec", - "source": "https://example.com/accelerator-spec" - } - }, - "scale_up_network": { - "method": "declared_tdp", - "switch_count": 1, - "tdp_per_switch": { - "value_w": 3500, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - } - } - ], - "scale_out": { - "present": false - } -} diff --git a/tests/fixtures/validation/submissions/sub_c/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_c/src/vllm/README.md deleted file mode 100644 index be3edfb30..000000000 --- a/tests/fixtures/validation/submissions/sub_c/src/vllm/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# vllm - -Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_d/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_d/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/config.yaml deleted file mode 100644 index b8581faa1..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/point.yaml deleted file mode 100644 index 9f37e80ab..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/point.yaml +++ /dev/null @@ -1,50 +0,0 @@ -concurrency: 1024 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -offline: elected -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1024 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 10240 - requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/result_summary.json deleted file mode 100644 index d9903b6b2..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 2448212914685.0, - "ttft": { - "total": 19499568439296.0, - "percentiles": { - "50": 793439471.0, - "95": 1030084586.4999999, - "90": 982755563.3999999 - } - }, - "output_sequence_lengths": { - "total": 7186180.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 2935.2758, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/system_desc.json deleted file mode 100644 index 2fc4ae5ce..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r1024/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-D", - "submitter_contact": "contact@example.com", - "system_name": "system_d", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 1.0 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/config.yaml deleted file mode 100644 index 51457c3cf..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/point.yaml deleted file mode 100644 index 08573ee57..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 128 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 128 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 1280 - requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/result_summary.json deleted file mode 100644 index 6d128c4d2..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 4981652524410.999, - "ttft": { - "total": 5200658755584.0, - "percentiles": { - "50": 211615346.5, - "95": 274691318.25, - "90": 262076123.9 - } - }, - "output_sequence_lengths": { - "total": 7208951.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1447.1003, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/system_desc.json deleted file mode 100644 index 4aa0a556b..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r128/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-D", - "submitter_contact": "contact@example.com", - "system_name": "system_d", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.493003 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/config.yaml deleted file mode 100644 index f9afe658a..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/point.yaml deleted file mode 100644 index afe70bf5d..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 16 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 16 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 160 - requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/result_summary.json deleted file mode 100644 index 9ba4737f5..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 21458438875063.0, - "ttft": { - "total": 3068488581120.0, - "percentiles": { - "50": 124857120.0, - "95": 181258587.5, - "90": 169978294.0 - } - }, - "output_sequence_lengths": { - "total": 7214114.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 336.1901, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/system_desc.json deleted file mode 100644 index a14bb3639..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r16/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-D", - "submitter_contact": "contact@example.com", - "system_name": "system_d", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.114534 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/config.yaml deleted file mode 100644 index f928653b5..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/point.yaml deleted file mode 100644 index 9295c1999..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 256 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 256 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 2560 - requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/result_summary.json deleted file mode 100644 index 8fdf6e29c..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 3735858007321.0, - "ttft": { - "total": 7190722621440.0, - "percentiles": { - "50": 292591252.5, - "95": 370424039.25, - "90": 354857481.9 - } - }, - "output_sequence_lengths": { - "total": 7211649.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1930.3863, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/system_desc.json deleted file mode 100644 index f3bcab6aa..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r256/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-D", - "submitter_contact": "contact@example.com", - "system_name": "system_d", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.657651 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/config.yaml deleted file mode 100644 index b03f20973..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 32 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/point.yaml deleted file mode 100644 index 0a689acc1..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 32 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 32 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 320 - requests_issued: 320 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/result_summary.json deleted file mode 100644 index bfb87ca1f..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 12616634802170.0, - "ttft": { - "total": 3350546350080.0, - "percentiles": { - "50": 136334080.0, - "95": 193898324.0, - "90": 182385475.2 - } - }, - "output_sequence_lengths": { - "total": 7189678.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 569.857, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/system_desc.json deleted file mode 100644 index d00bb00e3..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r32/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-D", - "submitter_contact": "contact@example.com", - "system_name": "system_d", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.194141 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/config.yaml deleted file mode 100644 index 846832e5e..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/point.yaml deleted file mode 100644 index 779d0205d..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 512 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 512 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 5120 - requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/result_summary.json deleted file mode 100644 index 4107afa33..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 2852962195525.0, - "ttft": { - "total": 10591377063936.0, - "percentiles": { - "50": 430964236.0, - "95": 646502413.0, - "90": 603394777.6 - } - }, - "output_sequence_lengths": { - "total": 7193097.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 2521.2732, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/system_desc.json deleted file mode 100644 index 17ad64718..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r512/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-D", - "submitter_contact": "contact@example.com", - "system_name": "system_d", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.858956 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/config.yaml deleted file mode 100644 index 7db6216f5..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/point.yaml deleted file mode 100644 index e67aff2c5..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 64 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 64 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 640 - requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/result_summary.json deleted file mode 100644 index 7916313e6..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 7605866967850.0, - "ttft": { - "total": 4213553332224.0, - "percentiles": { - "50": 171449924.0, - "95": 229212318.25, - "90": 217659839.4 - } - }, - "output_sequence_lengths": { - "total": 7198828.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 946.4836, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/system_desc.json deleted file mode 100644 index 3fa50bd10..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r64/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-D", - "submitter_contact": "contact@example.com", - "system_name": "system_d", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.322451 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/accuracy_results.json deleted file mode 100644 index 9806cebd4..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "humaneval": { - "dataset_name": "humaneval", - "num_samples": 500, - "score": 0.45, - "extractor": "ExactMatchExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/config.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/config.yaml deleted file mode 100644 index e1b56a7e8..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 8 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/point.yaml b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/point.yaml deleted file mode 100644 index 2ee5a330c..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 8 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: qwen3-coder-480b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 8 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 80 - requests_issued: 80 diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/result_summary.json deleted file mode 100644 index e82581895..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 40699327677953.0, - "ttft": { - "total": 3014703452160.0, - "percentiles": { - "50": 122668597.5, - "95": 178853483.5, - "90": 167616506.3 - } - }, - "output_sequence_lengths": { - "total": 7176538.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 176.3306, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/system_desc.json deleted file mode 100644 index 3b46b9774..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/qwen3-coder-480b/r8/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-D", - "submitter_contact": "contact@example.com", - "system_name": "system_d", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM (TPU)", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Custom TPU Host CPU", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "host_memory_capacity": "192 GB", - "host_networking": "Ethernet 100Gb", - "host_storage_capacity": "10 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Debian 11", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Google TPU v5p", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "qwen3-coder-480b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.060073 -} diff --git a/tests/fixtures/validation/submissions/sub_d/results/system_d/system_power.json b/tests/fixtures/validation/submissions/sub_d/results/system_d/system_power.json deleted file mode 100644 index f7b7ebfb4..000000000 --- a/tests/fixtures/validation/submissions/sub_d/results/system_d/system_power.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "system_desc_id": "system_d", - "cooling": "liquid", - "node_sets": [ - { - "node_set_id": 0, - "system_node_ensemble_id": 0, - "nodes_provisioned": 1, - "power_method": "component_sum", - "components": { - "cpu": { - "model": "Custom TPU Host CPU", - "count_per_node": 1, - "tdp_per_unit": { - "value_w": 350, - "source_type": "vendor_spec", - "source": "https://example.com/cpu-spec" - } - }, - "accelerator": { - "model": "Google TPU v5p", - "count_per_node": 8, - "tdp_per_unit": { - "value_w": 700, - "source_type": "vendor_spec", - "source": "https://example.com/accelerator-spec" - } - }, - "scale_up_network": { - "method": "declared_tdp", - "switch_count": 1, - "tdp_per_switch": { - "value_w": 3500, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - } - } - ], - "scale_out": { - "present": false - } -} diff --git a/tests/fixtures/validation/submissions/sub_d/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_d/src/vllm/README.md deleted file mode 100644 index be3edfb30..000000000 --- a/tests/fixtures/validation/submissions/sub_d/src/vllm/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# vllm - -Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_e/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_e/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/config.yaml deleted file mode 100644 index 6d8e711ba..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/point.yaml deleted file mode 100644 index e54fdc8c2..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 1 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 10 - requests_issued: 10 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/result_summary.json deleted file mode 100644 index 30d450bb4..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 25761392228349.0, - "ttft": { - "total": 1364073025980.0, - "percentiles": { - "50": 102040172.5, - "95": 239159702.6, - "90": 211735796.57999998 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 66.4212, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/system_desc.json deleted file mode 100644 index 788d18396..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-E", - "submitter_contact": "contact@example.com", - "system_name": "system_e", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.019947 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/config.yaml deleted file mode 100644 index b8581faa1..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/point.yaml deleted file mode 100644 index c0b85f7fc..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/point.yaml +++ /dev/null @@ -1,50 +0,0 @@ -concurrency: 1024 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -offline: elected -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1024 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 10240 - requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/result_summary.json deleted file mode 100644 index 64ae3c688..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 515488552187.99994, - "ttft": { - "total": 108213125776644.0, - "percentiles": { - "50": 8094937595.5, - "95": 17493931139.45, - "90": 15614132430.66 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 3319.3792, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/system_desc.json deleted file mode 100644 index 8cec9559c..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r1024/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-E", - "submitter_contact": "contact@example.com", - "system_name": "system_e", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.99684 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/config.yaml deleted file mode 100644 index 51457c3cf..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/point.yaml deleted file mode 100644 index 02b3bf995..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 128 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 128 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 1280 - requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/result_summary.json deleted file mode 100644 index 29712806d..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 627339011078.0, - "ttft": { - "total": 2788710324048.0, - "percentiles": { - "50": 208610886.0, - "95": 438744546.99999994, - "90": 392717814.79999995 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 2727.5555, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/system_desc.json deleted file mode 100644 index 70388ed1c..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r128/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-E", - "submitter_contact": "contact@example.com", - "system_name": "system_e", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.81911 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/config.yaml deleted file mode 100644 index f9afe658a..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/point.yaml deleted file mode 100644 index ccc4635a1..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 16 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 16 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 160 - requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/result_summary.json deleted file mode 100644 index f2614d79c..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 2064726029363.0, - "ttft": { - "total": 1680393412872.0, - "percentiles": { - "50": 125702679.0, - "95": 274517621.75, - "90": 244754633.2 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 828.7308, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/system_desc.json deleted file mode 100644 index 338f336ee..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r16/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-E", - "submitter_contact": "contact@example.com", - "system_name": "system_e", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.248875 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/config.yaml deleted file mode 100644 index ccc7a88ba..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 2 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/point.yaml deleted file mode 100644 index 3e2da1541..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 2 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 2 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 20 - requests_issued: 20 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/result_summary.json deleted file mode 100644 index 800a3470d..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 13163162954524.0, - "ttft": { - "total": 1484351746344.0, - "percentiles": { - "50": 111037683.0, - "95": 243860930.6, - "90": 217296281.07999998 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 129.9917, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/system_desc.json deleted file mode 100644 index 311fc4e89..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r2/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-E", - "submitter_contact": "contact@example.com", - "system_name": "system_e", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.039038 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/config.yaml deleted file mode 100644 index f928653b5..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/point.yaml deleted file mode 100644 index 8e3d1b5e7..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 256 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 256 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 2560 - requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/result_summary.json deleted file mode 100644 index 565ee9ee8..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 543519373006.00006, - "ttft": { - "total": 3961749503400.0, - "percentiles": { - "50": 296360675.0, - "95": 614244970.3, - "90": 550668111.24 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 3148.1895, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/system_desc.json deleted file mode 100644 index 166bcd6de..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r256/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-E", - "submitter_contact": "contact@example.com", - "system_name": "system_e", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.94543 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/config.yaml deleted file mode 100644 index b03f20973..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 32 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/point.yaml deleted file mode 100644 index b2ff8785c..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 32 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 32 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 320 - requests_issued: 320 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/result_summary.json deleted file mode 100644 index d2d18057a..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 1210809443938.0, - "ttft": { - "total": 1830336280008.0, - "percentiles": { - "50": 136919231.0, - "95": 299232559.5, - "90": 266769893.8 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1413.1885, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/system_desc.json deleted file mode 100644 index 829498fbd..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r32/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-E", - "submitter_contact": "contact@example.com", - "system_name": "system_e", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.424393 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/config.yaml deleted file mode 100644 index 24d4dedbd..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 4 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/point.yaml deleted file mode 100644 index 532ef974c..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 4 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 4 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 40 - requests_issued: 40 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/result_summary.json deleted file mode 100644 index 1e9c9a569..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 6914982314847.001, - "ttft": { - "total": 1477899507360.0, - "percentiles": { - "50": 110555020.0, - "95": 246967133.4, - "90": 219684710.72000003 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 247.4485, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/system_desc.json deleted file mode 100644 index c8cff3cce..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r4/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-E", - "submitter_contact": "contact@example.com", - "system_name": "system_e", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.074311 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/config.yaml deleted file mode 100644 index 846832e5e..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/point.yaml deleted file mode 100644 index 65e950b29..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 512 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 512 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 5120 - requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/result_summary.json deleted file mode 100644 index f5048b17f..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 513859639398.0, - "ttft": { - "total": 6792119454924.0, - "percentiles": { - "50": 508087930.5, - "95": 2020109026.3999984, - "90": 1717704807.2199988 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 3329.9015, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/system_desc.json deleted file mode 100644 index f7733fbdb..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r512/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-E", - "submitter_contact": "contact@example.com", - "system_name": "system_e", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 1.0 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/config.yaml deleted file mode 100644 index 7db6216f5..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/point.yaml deleted file mode 100644 index 148978304..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 64 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 64 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 640 - requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/result_summary.json deleted file mode 100644 index 06263731f..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 817907368671.0, - "ttft": { - "total": 2259057999168.0, - "percentiles": { - "50": 168989976.0, - "95": 370875724.7999998, - "90": 330498575.03999984 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 2092.0486, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/system_desc.json deleted file mode 100644 index 14c12df6f..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r64/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-E", - "submitter_contact": "contact@example.com", - "system_name": "system_e", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.628261 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/config.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/config.yaml deleted file mode 100644 index e1b56a7e8..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 8 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/point.yaml b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/point.yaml deleted file mode 100644 index 46a76d627..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 8 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 8 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 80 - requests_issued: 80 diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/result_summary.json deleted file mode 100644 index 2cbec8f84..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 3722968524345.0, - "ttft": { - "total": 1551415687080.0, - "percentiles": { - "50": 116054435.0, - "95": 253079730.75, - "90": 225674671.60000002 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 459.6069, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/system_desc.json deleted file mode 100644 index 659a7445e..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/llama3-8b/r8/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-E", - "submitter_contact": "contact@example.com", - "system_name": "system_e", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.138024 -} diff --git a/tests/fixtures/validation/submissions/sub_e/results/system_e/system_power.json b/tests/fixtures/validation/submissions/sub_e/results/system_e/system_power.json deleted file mode 100644 index 1cd446598..000000000 --- a/tests/fixtures/validation/submissions/sub_e/results/system_e/system_power.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "system_desc_id": "system_e", - "cooling": "air", - "node_sets": [ - { - "node_set_id": 0, - "system_node_ensemble_id": 0, - "nodes_provisioned": 1, - "power_method": "component_sum", - "components": { - "cpu": { - "model": "Intel Xeon Platinum 8480+", - "count_per_node": 2, - "tdp_per_unit": { - "value_w": 350, - "source_type": "vendor_spec", - "source": "https://example.com/cpu-spec" - } - }, - "accelerator": { - "model": "Intel Gaudi 3", - "count_per_node": 8, - "tdp_per_unit": { - "value_w": 700, - "source_type": "vendor_spec", - "source": "https://example.com/accelerator-spec" - } - }, - "scale_up_network": { - "method": "declared_tdp", - "switch_count": 1, - "tdp_per_switch": { - "value_w": 3500, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - } - } - ], - "scale_out": { - "present": false - } -} diff --git a/tests/fixtures/validation/submissions/sub_e/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_e/src/vllm/README.md deleted file mode 100644 index be3edfb30..000000000 --- a/tests/fixtures/validation/submissions/sub_e/src/vllm/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# vllm - -Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_f/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_f/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/config.yaml deleted file mode 100644 index 6d8e711ba..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/point.yaml deleted file mode 100644 index e54fdc8c2..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 1 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 10 - requests_issued: 10 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/result_summary.json deleted file mode 100644 index 4bb4b9092..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 45583460351935.0, - "ttft": { - "total": 2574627124512.0, - "percentiles": { - "50": 192596284.0, - "95": 449459125.5, - "90": 398086557.20000005 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 37.5378, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/system_desc.json deleted file mode 100644 index d9689a269..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-F", - "submitter_contact": "contact@example.com", - "system_name": "system_f", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 2, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 2, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 2, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.022095 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/config.yaml deleted file mode 100644 index b8581faa1..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/point.yaml deleted file mode 100644 index c0b85f7fc..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/point.yaml +++ /dev/null @@ -1,50 +0,0 @@ -concurrency: 1024 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -offline: elected -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1024 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 10240 - requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/result_summary.json deleted file mode 100644 index f96d2785c..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 1011902488209.0, - "ttft": { - "total": 479444018825568.0, - "percentiles": { - "50": 35865052276.0, - "95": 46774478776.74999, - "90": 44592593476.59999 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1690.9752, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/system_desc.json deleted file mode 100644 index 7cd1950a4..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r1024/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-F", - "submitter_contact": "contact@example.com", - "system_name": "system_f", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 2, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 2, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 2, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.995315 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/config.yaml deleted file mode 100644 index 51457c3cf..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/point.yaml deleted file mode 100644 index 02b3bf995..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 128 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 128 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 1280 - requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/result_summary.json deleted file mode 100644 index 9fd9647c8..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 1192600590769.0, - "ttft": { - "total": 5379052027176.0, - "percentiles": { - "50": 402382707.0, - "95": 869390356.9, - "90": 775988826.9200001 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1434.7653, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/system_desc.json deleted file mode 100644 index d2febe857..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r128/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-F", - "submitter_contact": "contact@example.com", - "system_name": "system_f", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 2, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 2, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 2, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.844509 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/config.yaml deleted file mode 100644 index f9afe658a..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/point.yaml deleted file mode 100644 index ccc4635a1..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 16 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 16 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 160 - requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/result_summary.json deleted file mode 100644 index 9c907163d..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 3505224837302.0005, - "ttft": { - "total": 3373550899608.0, - "percentiles": { - "50": 252360181.0, - "95": 550608223.3999999, - "90": 490958614.9199999 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 488.1576, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/system_desc.json deleted file mode 100644 index 2399da1c5..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r16/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-F", - "submitter_contact": "contact@example.com", - "system_name": "system_f", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 2, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 2, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 2, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.287332 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/config.yaml deleted file mode 100644 index ccc7a88ba..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 2 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/point.yaml deleted file mode 100644 index 3e2da1541..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 2 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 2 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 20 - requests_issued: 20 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/result_summary.json deleted file mode 100644 index 29b907f0d..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 23023578002485.996, - "ttft": { - "total": 2892442942776.0, - "percentiles": { - "50": 216370657.0, - "95": 457229317.34999996, - "90": 409057585.28 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 74.3196, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/system_desc.json deleted file mode 100644 index 9be0512be..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r2/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-F", - "submitter_contact": "contact@example.com", - "system_name": "system_f", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 2, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 2, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 2, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.043745 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/config.yaml deleted file mode 100644 index f928653b5..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/point.yaml deleted file mode 100644 index 8e3d1b5e7..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 256 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 256 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 2560 - requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/result_summary.json deleted file mode 100644 index 69e9f5c92..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 1045116285522.0, - "ttft": { - "total": 7725994840968.0, - "percentiles": { - "50": 577946951.0, - "95": 1242402278.2499998, - "90": 1109511212.7999997 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1637.236, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/system_desc.json deleted file mode 100644 index efac285be..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r256/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-F", - "submitter_contact": "contact@example.com", - "system_name": "system_f", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 2, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 2, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 2, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.963684 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/config.yaml deleted file mode 100644 index b03f20973..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 32 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/point.yaml deleted file mode 100644 index b2ff8785c..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 32 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 32 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 320 - requests_issued: 320 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/result_summary.json deleted file mode 100644 index 744dda378..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 2182110754084.9998, - "ttft": { - "total": 3596106784332.0, - "percentiles": { - "50": 269008586.5, - "95": 604284570.0, - "90": 537229373.3 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 784.1499, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/system_desc.json deleted file mode 100644 index a8747d0c0..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r32/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-F", - "submitter_contact": "contact@example.com", - "system_name": "system_f", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 2, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 2, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 2, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.461554 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/config.yaml deleted file mode 100644 index 24d4dedbd..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 4 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/point.yaml deleted file mode 100644 index 532ef974c..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 4 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 4 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 40 - requests_issued: 40 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/result_summary.json deleted file mode 100644 index 8e9864bd0..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 11622328296428.0, - "ttft": { - "total": 2922014582988.0, - "percentiles": { - "50": 218582778.5, - "95": 470498090.7499999, - "90": 420115028.29999995 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 147.2254, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/system_desc.json deleted file mode 100644 index 7f555ed59..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r4/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-F", - "submitter_contact": "contact@example.com", - "system_name": "system_f", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 2, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 2, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 2, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.086657 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/config.yaml deleted file mode 100644 index 846832e5e..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/point.yaml deleted file mode 100644 index 65e950b29..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 512 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 512 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 5120 - requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/result_summary.json deleted file mode 100644 index a2c730ba8..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 1007161846898.0, - "ttft": { - "total": 19342596948528.0, - "percentiles": { - "50": 1446932746.0, - "95": 17416985904.85, - "90": 14222975273.08 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1698.9345, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/system_desc.json deleted file mode 100644 index ddb8e4f05..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r512/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-F", - "submitter_contact": "contact@example.com", - "system_name": "system_f", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 2, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 2, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 2, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 1.0 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/config.yaml deleted file mode 100644 index 7db6216f5..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/point.yaml deleted file mode 100644 index 148978304..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 64 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 64 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 640 - requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/result_summary.json deleted file mode 100644 index 810894306..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 1528920443390.0, - "ttft": { - "total": 4340911548228.0, - "percentiles": { - "50": 324724083.5, - "95": 700762596.05, - "90": 625554893.54 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1119.157, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/system_desc.json deleted file mode 100644 index abb424f50..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r64/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-F", - "submitter_contact": "contact@example.com", - "system_name": "system_f", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 2, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 2, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 2, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.658741 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/accuracy_results.json deleted file mode 100644 index cd085ffb4..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/accuracy_results.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 13368, - "score": { - "rouge1": 38.7792, - "rouge2": 15.9075, - "rougel": 24.4957, - "rougelsum": 35.793, - "gen_len": 8167644.0 - }, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/config.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/config.yaml deleted file mode 100644 index e1b56a7e8..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 8 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/point.yaml b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/point.yaml deleted file mode 100644 index 46a76d627..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 8 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1024 -model_name: llama3-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 8 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 80 - requests_issued: 80 diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/result_summary.json deleted file mode 100644 index 667db8712..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 13368, - "n_samples_completed": 13368, - "n_samples_failed": 0, - "duration_ns": 6211960687374.0, - "ttft": { - "total": 3152037458208.0, - "percentiles": { - "50": 235789756.0, - "95": 509306917.05, - "90": 454603484.84000003 - } - }, - "output_sequence_lengths": { - "total": 1711102.0, - "percentiles": {} - }, - "tpot": { - "total": 54127032000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 275.4528, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/system_desc.json deleted file mode 100644 index a8e80717c..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/llama3-8b/r8/system_desc.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "submitter_org_names": "Submitter-F", - "submitter_contact": "contact@example.com", - "system_name": "system_f", - "system_category": "datacenter", - "max_supported_concurrency": 1024, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 2, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "512 GB", - "host_networking": "Ethernet 200Gb", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "Intel Gaudi 3", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "96 GB HBM2e", - "accelerator_memory_type": "HBM3e" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 2, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 2, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.162133 -} diff --git a/tests/fixtures/validation/submissions/sub_f/results/system_f/system_power.json b/tests/fixtures/validation/submissions/sub_f/results/system_f/system_power.json deleted file mode 100644 index 21550a655..000000000 --- a/tests/fixtures/validation/submissions/sub_f/results/system_f/system_power.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "system_desc_id": "system_f", - "cooling": "air", - "node_sets": [ - { - "node_set_id": 0, - "system_node_ensemble_id": 0, - "nodes_provisioned": 2, - "power_method": "component_sum", - "components": { - "cpu": { - "model": "Intel Xeon Platinum 8480+", - "count_per_node": 2, - "tdp_per_unit": { - "value_w": 350, - "source_type": "vendor_spec", - "source": "https://example.com/cpu-spec" - } - }, - "accelerator": { - "model": "Intel Gaudi 3", - "count_per_node": 8, - "tdp_per_unit": { - "value_w": 700, - "source_type": "vendor_spec", - "source": "https://example.com/accelerator-spec" - } - }, - "scale_up_network": { - "method": "declared_tdp", - "switch_count": 1, - "tdp_per_switch": { - "value_w": 3500, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - } - } - ], - "scale_out": { - "present": true, - "cabling": "passive", - "required_bandwidth_tbps": 6.4, - "nics": { - "count_per_node": 8, - "bandwidth_per_nic_gbps": 400, - "counted": true, - "tdp_per_nic": { - "value_w": 25, - "source_type": "vendor_spec", - "source": "https://example.com/nic-spec" - } - }, - "switches": [ - { - "model": "NVIDIA Spectrum SN5610", - "count": 1, - "bandwidth_tbps": 51.2, - "power_per_switch": { - "value_w": 900, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - ] - } -} diff --git a/tests/fixtures/validation/submissions/sub_f/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_f/src/vllm/README.md deleted file mode 100644 index be3edfb30..000000000 --- a/tests/fixtures/validation/submissions/sub_f/src/vllm/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# vllm - -Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_g/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_g/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/config.yaml deleted file mode 100644 index b8581faa1..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/point.yaml deleted file mode 100644 index 402f06b6f..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 1024 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1024 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 10240 - requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/result_summary.json deleted file mode 100644 index 0e7573814..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 358433737274.0, - "ttft": { - "total": 5094461190144.0, - "percentiles": { - "50": 207294156.5, - "95": 475022456.0, - "90": 421476796.1 - } - }, - "output_sequence_lengths": { - "total": 5972600.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 16663.052, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/system_desc.json deleted file mode 100644 index e1b7473e2..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1024/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-G", - "submitter_contact": "contact@example.com", - "system_name": "system_g", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.979024 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/config.yaml deleted file mode 100644 index 51457c3cf..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/point.yaml deleted file mode 100644 index d3e80b258..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 128 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 128 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 1280 - requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/result_summary.json deleted file mode 100644 index 7bb8379f6..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 1154776483445.0, - "ttft": { - "total": 2436995543040.0, - "percentiles": { - "50": 99161602.5, - "95": 649174965.25, - "90": 539172292.7 - } - }, - "output_sequence_lengths": { - "total": 5954161.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 5156.1156, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/system_desc.json deleted file mode 100644 index d2cf70a38..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r128/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-G", - "submitter_contact": "contact@example.com", - "system_name": "system_g", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.302943 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/config.yaml deleted file mode 100644 index 087256dee..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1536 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/point.yaml deleted file mode 100644 index 384d2fa43..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 1536 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1536 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 15360 - requests_issued: 15360 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/result_summary.json deleted file mode 100644 index c57070abe..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 371427380557.0, - "ttft": { - "total": 6655861899264.0, - "percentiles": { - "50": 270827714.0, - "95": 2812902801.5, - "90": 2304487784.0 - } - }, - "output_sequence_lengths": { - "total": 5972600.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 16080.1285, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/system_desc.json deleted file mode 100644 index 720ed8eed..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r1536/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-G", - "submitter_contact": "contact@example.com", - "system_name": "system_g", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.944775 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/config.yaml deleted file mode 100644 index 5ace5b44d..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 192 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/point.yaml deleted file mode 100644 index d0968cfb8..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 192 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 192 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 1920 - requests_issued: 1920 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/result_summary.json deleted file mode 100644 index 1a0978c61..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 1008814134637.0, - "ttft": { - "total": 2930111361024.0, - "percentiles": { - "50": 119226536.5, - "95": 610194947.75, - "90": 512001265.5 - } - }, - "output_sequence_lengths": { - "total": 5970998.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 5918.8286, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/system_desc.json deleted file mode 100644 index 776439f6c..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r192/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-G", - "submitter_contact": "contact@example.com", - "system_name": "system_g", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.347756 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/config.yaml deleted file mode 100644 index 6b53d1bba..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 2048 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/point.yaml deleted file mode 100644 index a1ed6ee7e..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/point.yaml +++ /dev/null @@ -1,50 +0,0 @@ -concurrency: 2048 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -offline: elected -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 2048 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 20480 - requests_issued: 20480 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/result_summary.json deleted file mode 100644 index bb4623007..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 350915227928.0, - "ttft": { - "total": 7688238305280.0, - "percentiles": { - "50": 312835217.5, - "95": 4378115873.25, - "90": 3565059742.1000004 - } - }, - "output_sequence_lengths": { - "total": 5972600.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 17020.065, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/system_desc.json deleted file mode 100644 index 093b5cb06..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r2048/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-G", - "submitter_contact": "contact@example.com", - "system_name": "system_g", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 1.0 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/config.yaml deleted file mode 100644 index f928653b5..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/point.yaml deleted file mode 100644 index d7a6a9b5c..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 256 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 256 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 2560 - requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/result_summary.json deleted file mode 100644 index 36daea8a5..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 776418597591.0, - "ttft": { - "total": 3292125204480.0, - "percentiles": { - "50": 133956917.5, - "95": 590184585.25, - "90": 498939051.70000005 - } - }, - "output_sequence_lengths": { - "total": 5962724.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 7679.78, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/system_desc.json deleted file mode 100644 index a44a44683..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r256/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-G", - "submitter_contact": "contact@example.com", - "system_name": "system_g", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.451219 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/config.yaml deleted file mode 100644 index 05556e355..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 384 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/point.yaml deleted file mode 100644 index 22cad9262..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 384 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 384 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 3840 - requests_issued: 3840 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/result_summary.json deleted file mode 100644 index ea56360b8..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 517816400500.0, - "ttft": { - "total": 3370839257088.0, - "percentiles": { - "50": 137159800.5, - "95": 434521706.0, - "90": 375049324.9 - } - }, - "output_sequence_lengths": { - "total": 5964063.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 11517.7175, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/system_desc.json deleted file mode 100644 index 8365da8b4..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r384/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-G", - "submitter_contact": "contact@example.com", - "system_name": "system_g", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.676714 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/config.yaml deleted file mode 100644 index 846832e5e..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/point.yaml deleted file mode 100644 index d8a2668be..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 512 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 512 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 5120 - requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/result_summary.json deleted file mode 100644 index ee10d7260..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 444594042812.0, - "ttft": { - "total": 3720757702656.0, - "percentiles": { - "50": 151398018.5, - "95": 409875781.5, - "90": 358180228.9 - } - }, - "output_sequence_lengths": { - "total": 5961100.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 13407.9619, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/system_desc.json deleted file mode 100644 index 1ef2810de..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r512/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-G", - "submitter_contact": "contact@example.com", - "system_name": "system_g", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.787774 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/config.yaml deleted file mode 100644 index 7db6216f5..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/point.yaml deleted file mode 100644 index b7140d590..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 64 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 64 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 640 - requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/result_summary.json deleted file mode 100644 index 878b5a8d4..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 2145983999331.0002, - "ttft": { - "total": 2184279601152.0, - "percentiles": { - "50": 88878564.5, - "95": 958562205.25, - "90": 784625477.1 - } - }, - "output_sequence_lengths": { - "total": 5979513.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 2786.3735, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/system_desc.json deleted file mode 100644 index 26b278e07..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r64/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-G", - "submitter_contact": "contact@example.com", - "system_name": "system_g", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.163711 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/accuracy_results.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/config.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/config.yaml deleted file mode 100644 index 240492592..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 768 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/point.yaml b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/point.yaml deleted file mode 100644 index 8949daa55..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 768 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 768 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 7680 - requests_issued: 7680 diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/result_summary.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/result_summary.json deleted file mode 100644 index 24c446ca5..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 388271410974.0, - "ttft": { - "total": 4313087397888.0, - "percentiles": { - "50": 175499975.5, - "95": 426849699.25, - "90": 376579754.5 - } - }, - "output_sequence_lengths": { - "total": 5972600.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 15382.5387, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/system_desc.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/system_desc.json deleted file mode 100644 index 2fc2d7a6e..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/llama3-70b/r768/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-G", - "submitter_contact": "contact@example.com", - "system_name": "system_g", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "vLLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.903788 -} diff --git a/tests/fixtures/validation/submissions/sub_g/results/system_g/system_power.json b/tests/fixtures/validation/submissions/sub_g/results/system_g/system_power.json deleted file mode 100644 index 1ab0e9142..000000000 --- a/tests/fixtures/validation/submissions/sub_g/results/system_g/system_power.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "system_desc_id": "system_g", - "cooling": "air", - "node_sets": [ - { - "node_set_id": 0, - "system_node_ensemble_id": 0, - "nodes_provisioned": 1, - "power_method": "component_sum", - "components": { - "cpu": { - "model": "Intel Xeon Platinum 8480+", - "count_per_node": 2, - "tdp_per_unit": { - "value_w": 350, - "source_type": "vendor_spec", - "source": "https://example.com/cpu-spec" - } - }, - "accelerator": { - "model": "NVIDIA H100-SXM5-80GB", - "count_per_node": 8, - "tdp_per_unit": { - "value_w": 700, - "source_type": "vendor_spec", - "source": "https://example.com/accelerator-spec" - } - }, - "scale_up_network": { - "method": "declared_tdp", - "switch_count": 1, - "tdp_per_switch": { - "value_w": 3500, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - } - } - ], - "scale_out": { - "present": false - } -} diff --git a/tests/fixtures/validation/submissions/sub_g/src/vllm/README.md b/tests/fixtures/validation/submissions/sub_g/src/vllm/README.md deleted file mode 100644 index be3edfb30..000000000 --- a/tests/fixtures/validation/submissions/sub_g/src/vllm/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# vllm - -Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_h/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_h/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/config.yaml deleted file mode 100644 index b8581faa1..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/point.yaml deleted file mode 100644 index 402f06b6f..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 1024 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1024 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 10240 - requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/result_summary.json deleted file mode 100644 index fcbea3f4d..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 392249945920.0, - "ttft": { - "total": 3834814304256.0, - "percentiles": { - "50": 156038993.5, - "95": 510605960.25, - "90": 439692566.90000004 - } - }, - "output_sequence_lengths": { - "total": 5958417.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 15190.3577, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/system_desc.json deleted file mode 100644 index da1ccbf2b..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1024/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-H", - "submitter_contact": "contact@example.com", - "system_name": "system_h", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "SGLang", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 1.0 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/config.yaml deleted file mode 100644 index 51457c3cf..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/point.yaml deleted file mode 100644 index d3e80b258..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 128 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 128 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 1280 - requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/result_summary.json deleted file mode 100644 index 842895879..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 1403030676854.0, - "ttft": { - "total": 2718343397376.0, - "percentiles": { - "50": 110609676.0, - "95": 758335350.75, - "90": 628790215.8 - } - }, - "output_sequence_lengths": { - "total": 5949986.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 4240.8096, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/system_desc.json deleted file mode 100644 index 8651c6c07..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r128/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-H", - "submitter_contact": "contact@example.com", - "system_name": "system_h", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "SGLang", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.279178 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/config.yaml deleted file mode 100644 index 087256dee..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1536 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/point.yaml deleted file mode 100644 index 384d2fa43..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 1536 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1536 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 15360 - requests_issued: 15360 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/result_summary.json deleted file mode 100644 index 6de815aba..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 402038739154.0, - "ttft": { - "total": 4694270042112.0, - "percentiles": { - "50": 191010337.0, - "95": 2130321446.0, - "90": 1742459224.2 - } - }, - "output_sequence_lengths": { - "total": 5958417.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 14820.5046, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/system_desc.json deleted file mode 100644 index 8bd4f491b..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r1536/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-H", - "submitter_contact": "contact@example.com", - "system_name": "system_h", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "SGLang", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.975652 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/config.yaml deleted file mode 100644 index 5ace5b44d..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 192 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/point.yaml deleted file mode 100644 index d0968cfb8..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 192 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 192 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 1920 - requests_issued: 1920 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/result_summary.json deleted file mode 100644 index 9410fd05f..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 1001832049974.0001, - "ttft": { - "total": 2768901292032.0, - "percentiles": { - "50": 112666882.0, - "95": 635672662.0, - "90": 531071506.0 - } - }, - "output_sequence_lengths": { - "total": 5956127.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 5945.235, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/system_desc.json deleted file mode 100644 index ff9566c99..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r192/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-H", - "submitter_contact": "contact@example.com", - "system_name": "system_h", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "SGLang", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.391382 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/config.yaml deleted file mode 100644 index 6b53d1bba..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 2048 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/point.yaml deleted file mode 100644 index a1ed6ee7e..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/point.yaml +++ /dev/null @@ -1,50 +0,0 @@ -concurrency: 2048 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -offline: elected -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 2048 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 20480 - requests_issued: 20480 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/result_summary.json deleted file mode 100644 index abd1561a7..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 409541274676.0, - "ttft": { - "total": 5435139072000.0, - "percentiles": { - "50": 221156375.0, - "95": 4408367949.0, - "90": 3570925634.2000003 - } - }, - "output_sequence_lengths": { - "total": 5958417.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 14549.0024, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/system_desc.json deleted file mode 100644 index 6d37f526e..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r2048/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-H", - "submitter_contact": "contact@example.com", - "system_name": "system_h", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "SGLang", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.957779 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/config.yaml deleted file mode 100644 index f928653b5..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/point.yaml deleted file mode 100644 index d7a6a9b5c..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 256 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 256 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 2560 - requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/result_summary.json deleted file mode 100644 index 7129f0901..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 799717038355.0, - "ttft": { - "total": 2881879240704.0, - "percentiles": { - "50": 117263966.5, - "95": 558828541.0, - "90": 470515626.1 - } - }, - "output_sequence_lengths": { - "total": 5960490.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 7453.2487, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/system_desc.json deleted file mode 100644 index af9605513..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r256/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-H", - "submitter_contact": "contact@example.com", - "system_name": "system_h", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "SGLang", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.490657 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/config.yaml deleted file mode 100644 index 05556e355..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 384 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/point.yaml deleted file mode 100644 index 22cad9262..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 384 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 384 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 3840 - requests_issued: 3840 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/result_summary.json deleted file mode 100644 index 2cdfd5d7c..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 600137480188.0, - "ttft": { - "total": 3042139607040.0, - "percentiles": { - "50": 123784977.5, - "95": 483974886.0, - "90": 411936904.3 - } - }, - "output_sequence_lengths": { - "total": 5958391.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 9928.3767, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/system_desc.json deleted file mode 100644 index 065fc68ec..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r384/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-H", - "submitter_contact": "contact@example.com", - "system_name": "system_h", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "SGLang", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.653597 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/config.yaml deleted file mode 100644 index 846832e5e..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/point.yaml deleted file mode 100644 index d8a2668be..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 512 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 512 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 5120 - requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/result_summary.json deleted file mode 100644 index 5bbd690dd..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 500684255434.00006, - "ttft": { - "total": 3142710755328.0, - "percentiles": { - "50": 127877228.0, - "95": 443205541.0, - "90": 380139878.4 - } - }, - "output_sequence_lengths": { - "total": 5958417.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 11900.548, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/system_desc.json deleted file mode 100644 index adf7eac1a..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r512/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-H", - "submitter_contact": "contact@example.com", - "system_name": "system_h", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "SGLang", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.783428 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/config.yaml deleted file mode 100644 index 7db6216f5..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/point.yaml deleted file mode 100644 index b7140d590..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 64 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 64 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 640 - requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/result_summary.json deleted file mode 100644 index 1ecf29e95..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 2608958278520.0, - "ttft": { - "total": 2562863296512.0, - "percentiles": { - "50": 104283174.5, - "95": 1100174349.75, - "90": 900996114.7 - } - }, - "output_sequence_lengths": { - "total": 5965062.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 2286.3769, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/system_desc.json deleted file mode 100644 index d282fbc88..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r64/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-H", - "submitter_contact": "contact@example.com", - "system_name": "system_h", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "SGLang", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.150515 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/accuracy_results.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/accuracy_results.json deleted file mode 100644 index 7242aa745..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/accuracy_results.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "cnn_dailymail_validation": { - "dataset_name": "cnn_dailymail_validation", - "num_samples": 500, - "score": 0.45, - "extractor": "RougeExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/config.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/config.yaml deleted file mode 100644 index 240492592..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 768 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/point.yaml b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/point.yaml deleted file mode 100644 index 8949daa55..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 768 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 2048 -model_name: llama3-70b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 768 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 7680 - requests_issued: 7680 diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/result_summary.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/result_summary.json deleted file mode 100644 index 643f6985b..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 24576, - "n_samples_completed": 24576, - "n_samples_failed": 0, - "duration_ns": 513149916081.0, - "ttft": { - "total": 3919180652544.0, - "percentiles": { - "50": 159471869.0, - "95": 529036573.0, - "90": 455123632.2 - } - }, - "output_sequence_lengths": { - "total": 5958417.0, - "percentiles": {} - }, - "tpot": { - "total": 99508224000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 11611.4547, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/system_desc.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/system_desc.json deleted file mode 100644 index 6512e5184..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/llama3-70b/r768/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-H", - "submitter_contact": "contact@example.com", - "system_name": "system_h", - "system_category": "datacenter", - "max_supported_concurrency": 2048, - "serving_framework": "SGLang", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8480+", - "host_processors_per_node": 2, - "host_processor_core_count": 60, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100-SXM5-80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3-70b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.764396 -} diff --git a/tests/fixtures/validation/submissions/sub_h/results/system_h/system_power.json b/tests/fixtures/validation/submissions/sub_h/results/system_h/system_power.json deleted file mode 100644 index f5baab29a..000000000 --- a/tests/fixtures/validation/submissions/sub_h/results/system_h/system_power.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "system_desc_id": "system_h", - "cooling": "air", - "node_sets": [ - { - "node_set_id": 0, - "system_node_ensemble_id": 0, - "nodes_provisioned": 1, - "power_method": "component_sum", - "components": { - "cpu": { - "model": "Intel Xeon Platinum 8480+", - "count_per_node": 2, - "tdp_per_unit": { - "value_w": 350, - "source_type": "vendor_spec", - "source": "https://example.com/cpu-spec" - } - }, - "accelerator": { - "model": "NVIDIA H100-SXM5-80GB", - "count_per_node": 8, - "tdp_per_unit": { - "value_w": 700, - "source_type": "vendor_spec", - "source": "https://example.com/accelerator-spec" - } - }, - "scale_up_network": { - "method": "declared_tdp", - "switch_count": 1, - "tdp_per_switch": { - "value_w": 3500, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - } - } - ], - "scale_out": { - "present": false - } -} diff --git a/tests/fixtures/validation/submissions/sub_h/src/sglang/README.md b/tests/fixtures/validation/submissions/sub_h/src/sglang/README.md deleted file mode 100644 index 306956568..000000000 --- a/tests/fixtures/validation/submissions/sub_h/src/sglang/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# sglang - -Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_i/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_i/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/config.yaml deleted file mode 100644 index 6d8e711ba..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/point.yaml deleted file mode 100644 index 0033075a4..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 1 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 10 - requests_issued: 10 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/result_summary.json deleted file mode 100644 index 57e255517..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 29, - "n_samples_completed": 29, - "n_samples_failed": 0, - "duration_ns": 1810326781531.0, - "ttft": { - "total": 3239905752.0, - "percentiles": { - "50": 111720888.0, - "95": 121428535.79999998, - "90": 119487006.23999998 - } - }, - "output_sequence_lengths": { - "total": 168384.0, - "percentiles": {} - }, - "tpot": { - "total": 117421000.00000001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 93.013, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/system_desc.json deleted file mode 100644 index bd28e2c75..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r1/system_desc.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "submitter_org_names": "Submitter-I", - "submitter_contact": "contact@example.com", - "system_name": "system_i", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8562Y+", - "host_processors_per_node": 2, - "host_processor_core_count": 32, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H200-SXM-141GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "141 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.01422 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/config.yaml deleted file mode 100644 index 51457c3cf..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/point.yaml deleted file mode 100644 index 07b71d3b6..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 128 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 128 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 1280 - requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/result_summary.json deleted file mode 100644 index 62ee53f3c..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 1675, - "n_samples_completed": 1675, - "n_samples_failed": 0, - "duration_ns": 2363311856036.0, - "ttft": { - "total": 268304677475.0, - "percentiles": { - "50": 160181897.0, - "95": 456279775.1, - "90": 397060199.48 - } - }, - "output_sequence_lengths": { - "total": 6215384.0, - "percentiles": {} - }, - "tpot": { - "total": 6782075000.000001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 2629.9466, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/system_desc.json deleted file mode 100644 index 00858a98f..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r128/system_desc.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "submitter_org_names": "Submitter-I", - "submitter_contact": "contact@example.com", - "system_name": "system_i", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8562Y+", - "host_processors_per_node": 2, - "host_processor_core_count": 32, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H200-SXM-141GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "141 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.402063 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/config.yaml deleted file mode 100644 index f9afe658a..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 16 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/point.yaml deleted file mode 100644 index d1d3b5d43..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 16 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 16 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 160 - requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/result_summary.json deleted file mode 100644 index d857471f2..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 339, - "n_samples_completed": 339, - "n_samples_failed": 0, - "duration_ns": 2003156043150.0, - "ttft": { - "total": 45912856545.0, - "percentiles": { - "50": 135436155.0, - "95": 241107175.0999999, - "90": 219972971.07999992 - } - }, - "output_sequence_lengths": { - "total": 1378214.0, - "percentiles": {} - }, - "tpot": { - "total": 1372611000.0000002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 688.0213, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/system_desc.json deleted file mode 100644 index 1b5895a94..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r16/system_desc.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "submitter_org_names": "Submitter-I", - "submitter_contact": "contact@example.com", - "system_name": "system_i", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8562Y+", - "host_processors_per_node": 2, - "host_processor_core_count": 32, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H200-SXM-141GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "141 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.105184 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/config.yaml deleted file mode 100644 index ccc7a88ba..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 2 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/point.yaml deleted file mode 100644 index a181e0641..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 2 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 2 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 20 - requests_issued: 20 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/result_summary.json deleted file mode 100644 index 0d10e7170..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 68, - "n_samples_completed": 68, - "n_samples_failed": 0, - "duration_ns": 1944103059102.0002, - "ttft": { - "total": 8684697316.0, - "percentiles": { - "50": 127716137.0, - "95": 137065360.35, - "90": 135195515.68 - } - }, - "output_sequence_lengths": { - "total": 315398.0, - "percentiles": {} - }, - "tpot": { - "total": 275332000.00000006, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 162.2332, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/system_desc.json deleted file mode 100644 index 58c9bd606..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r2/system_desc.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "submitter_org_names": "Submitter-I", - "submitter_contact": "contact@example.com", - "system_name": "system_i", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8562Y+", - "host_processors_per_node": 2, - "host_processor_core_count": 32, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H200-SXM-141GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "141 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.024802 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/config.yaml deleted file mode 100644 index f928653b5..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/point.yaml deleted file mode 100644 index 51eff9077..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 256 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 256 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 2560 - requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/result_summary.json deleted file mode 100644 index 381826878..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 2759, - "n_samples_completed": 2759, - "n_samples_failed": 0, - "duration_ns": 2418594571195.9995, - "ttft": { - "total": 512378878272.0, - "percentiles": { - "50": 185711808.0, - "95": 842228387.0, - "90": 710925071.2 - } - }, - "output_sequence_lengths": { - "total": 10495599.0, - "percentiles": {} - }, - "tpot": { - "total": 11171191000.000002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 4339.5446, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/system_desc.json deleted file mode 100644 index 04a8e2c14..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r256/system_desc.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "submitter_org_names": "Submitter-I", - "submitter_contact": "contact@example.com", - "system_name": "system_i", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8562Y+", - "host_processors_per_node": 2, - "host_processor_core_count": 32, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H200-SXM-141GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "141 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.663424 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/config.yaml deleted file mode 100644 index b03f20973..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 32 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/point.yaml deleted file mode 100644 index 8d7fd6807..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 32 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 32 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 320 - requests_issued: 320 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/result_summary.json deleted file mode 100644 index 000263e0d..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 516, - "n_samples_completed": 516, - "n_samples_failed": 0, - "duration_ns": 2102160301502.9998, - "ttft": { - "total": 72592828674.0, - "percentiles": { - "50": 140683776.5, - "95": 237608990.0, - "90": 218223947.3 - } - }, - "output_sequence_lengths": { - "total": 1986475.0, - "percentiles": {} - }, - "tpot": { - "total": 2089284000.0000002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 944.9684, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/system_desc.json deleted file mode 100644 index fa5657e0d..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r32/system_desc.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "submitter_org_names": "Submitter-I", - "submitter_contact": "contact@example.com", - "system_name": "system_i", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8562Y+", - "host_processors_per_node": 2, - "host_processor_core_count": 32, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H200-SXM-141GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "141 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.144466 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/config.yaml deleted file mode 100644 index 24d4dedbd..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 4 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/point.yaml deleted file mode 100644 index 63c0feabc..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 4 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 4 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 40 - requests_issued: 40 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/result_summary.json deleted file mode 100644 index ee8574091..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 117, - "n_samples_completed": 117, - "n_samples_failed": 0, - "duration_ns": 1962849595746.0, - "ttft": { - "total": 14924097162.0, - "percentiles": { - "50": 127556386.0, - "95": 135825117.4, - "90": 134171371.12 - } - }, - "output_sequence_lengths": { - "total": 539431.0, - "percentiles": {} - }, - "tpot": { - "total": 473733000.00000006, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 274.8203, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/system_desc.json deleted file mode 100644 index 6a6d98ddf..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r4/system_desc.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "submitter_org_names": "Submitter-I", - "submitter_contact": "contact@example.com", - "system_name": "system_i", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8562Y+", - "host_processors_per_node": 2, - "host_processor_core_count": 32, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H200-SXM-141GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "141 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.042014 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/config.yaml deleted file mode 100644 index 846832e5e..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/point.yaml deleted file mode 100644 index ac8f266dd..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/point.yaml +++ /dev/null @@ -1,50 +0,0 @@ -concurrency: 512 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -offline: elected -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 512 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 5120 - requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/result_summary.json deleted file mode 100644 index e5a706061..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 4388, - "n_samples_completed": 4388, - "n_samples_failed": 0, - "duration_ns": 2594198991881.0, - "ttft": { - "total": 1184049108896.0, - "percentiles": { - "50": 269837992.0, - "95": 2367829187.25, - "90": 1948230948.2 - } - }, - "output_sequence_lengths": { - "total": 16968991.0, - "percentiles": {} - }, - "tpot": { - "total": 17767012000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 6541.1293, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/system_desc.json deleted file mode 100644 index dc06f8e7e..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r512/system_desc.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "submitter_org_names": "Submitter-I", - "submitter_contact": "contact@example.com", - "system_name": "system_i", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8562Y+", - "host_processors_per_node": 2, - "host_processor_core_count": 32, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H200-SXM-141GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "141 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 1.0 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/config.yaml deleted file mode 100644 index 7db6216f5..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/point.yaml deleted file mode 100644 index 9f4a04371..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 64 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 64 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 640 - requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/result_summary.json deleted file mode 100644 index 8463177f0..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 906, - "n_samples_completed": 906, - "n_samples_failed": 0, - "duration_ns": 2209290005442.0, - "ttft": { - "total": 134375863380.0, - "percentiles": { - "50": 148317730.0, - "95": 300972722.5, - "90": 270441724.0 - } - }, - "output_sequence_lengths": { - "total": 3400893.0, - "percentiles": {} - }, - "tpot": { - "total": 3668394000.0000005, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1539.3602, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/system_desc.json deleted file mode 100644 index 4aa7cc486..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r64/system_desc.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "submitter_org_names": "Submitter-I", - "submitter_contact": "contact@example.com", - "system_name": "system_i", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8562Y+", - "host_processors_per_node": 2, - "host_processor_core_count": 32, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H200-SXM-141GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "141 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.235336 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/accuracy_results.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/config.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/config.yaml deleted file mode 100644 index e1b56a7e8..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 8 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/point.yaml b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/point.yaml deleted file mode 100644 index ff5648af6..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 8 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 512 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 8 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 80 - requests_issued: 80 diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/result_summary.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/result_summary.json deleted file mode 100644 index cac202f46..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 220, - "n_samples_completed": 220, - "n_samples_failed": 0, - "duration_ns": 1919705564758.0, - "ttft": { - "total": 28929776150.000004, - "percentiles": { - "50": 131498982.50000001, - "95": 139797791.85, - "90": 138138029.98 - } - }, - "output_sequence_lengths": { - "total": 867074.0, - "percentiles": {} - }, - "tpot": { - "total": 890780000.0000001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 451.6703, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/system_desc.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/system_desc.json deleted file mode 100644 index 61e1e3398..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/deepseek-r1/r8/system_desc.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "submitter_org_names": "Submitter-I", - "submitter_contact": "contact@example.com", - "system_name": "system_i", - "system_category": "datacenter", - "max_supported_concurrency": 512, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "Intel Xeon Platinum 8562Y+", - "host_processors_per_node": 2, - "host_processor_core_count": 32, - "host_memory_capacity": "2 TiB", - "host_networking": "InfiniBand NDR", - "host_storage_capacity": "15 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H200-SXM-141GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "141 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "18x NVLink 4th Gen 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Air-cooled", - "tps_utilization": 0.069051 -} diff --git a/tests/fixtures/validation/submissions/sub_i/results/system_i/system_power.json b/tests/fixtures/validation/submissions/sub_i/results/system_i/system_power.json deleted file mode 100644 index 4d902dc7d..000000000 --- a/tests/fixtures/validation/submissions/sub_i/results/system_i/system_power.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "system_desc_id": "system_i", - "cooling": "air", - "node_sets": [ - { - "node_set_id": 0, - "system_node_ensemble_id": 0, - "nodes_provisioned": 1, - "power_method": "component_sum", - "components": { - "cpu": { - "model": "Intel Xeon Platinum 8562Y+", - "count_per_node": 2, - "tdp_per_unit": { - "value_w": 350, - "source_type": "vendor_spec", - "source": "https://example.com/cpu-spec" - } - }, - "accelerator": { - "model": "NVIDIA H200-SXM-141GB", - "count_per_node": 8, - "tdp_per_unit": { - "value_w": 700, - "source_type": "vendor_spec", - "source": "https://example.com/accelerator-spec" - } - }, - "scale_up_network": { - "method": "declared_tdp", - "switch_count": 1, - "tdp_per_switch": { - "value_w": 3500, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - } - } - ], - "scale_out": { - "present": false - } -} diff --git a/tests/fixtures/validation/submissions/sub_i/src/tensorrt-llm/README.md b/tests/fixtures/validation/submissions/sub_i/src/tensorrt-llm/README.md deleted file mode 100644 index ab1543fd0..000000000 --- a/tests/fixtures/validation/submissions/sub_i/src/tensorrt-llm/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# tensorrt-llm - -Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/sub_j/docs/.gitkeep b/tests/fixtures/validation/submissions/sub_j/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/config.yaml deleted file mode 100644 index b8581faa1..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1024 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/point.yaml deleted file mode 100644 index b786427c8..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 1024 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 16384 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1024 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 10240 - requests_issued: 10240 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/result_summary.json deleted file mode 100644 index 5fa3c1f9e..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 8776, - "n_samples_completed": 8776, - "n_samples_failed": 0, - "duration_ns": 323142428984.0, - "ttft": { - "total": 3105883110512.0, - "percentiles": { - "50": 353906462.0, - "95": 1762244150.0, - "90": 1480576612.4 - } - }, - "output_sequence_lengths": { - "total": 34315466.0, - "percentiles": {} - }, - "tpot": { - "total": 35534024000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 106193.0063, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/system_desc.json deleted file mode 100644 index a91ea8c6b..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r1024/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-J", - "submitter_contact": "contact@example.com", - "system_name": "system_j", - "system_category": "datacenter", - "max_supported_concurrency": 16384, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", - "host_processors_per_node": 36, - "host_processor_core_count": 72, - "host_memory_capacity": "2.16 TB LPDDR5x", - "host_networking": "InfiniBand NDR400", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04 (aarch64)", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA GB300", - "accelerators_per_node": 72, - "accelerator_memory_capacity": "288 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 5.0" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 72, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.283152 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/config.yaml deleted file mode 100644 index 51457c3cf..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 128 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/point.yaml deleted file mode 100644 index 5c1c63043..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 128 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 16384 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 128 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 1280 - requests_issued: 1280 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/result_summary.json deleted file mode 100644 index e0332ec71..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 8776, - "n_samples_completed": 8776, - "n_samples_failed": 0, - "duration_ns": 1335987189179.0002, - "ttft": { - "total": 2152528849644.0, - "percentiles": { - "50": 245274481.5, - "95": 334071958.0, - "90": 316312462.7 - } - }, - "output_sequence_lengths": { - "total": 34063493.0, - "percentiles": {} - }, - "tpot": { - "total": 35534024000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 25496.871, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/system_desc.json deleted file mode 100644 index 038d7bbbc..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r128/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-J", - "submitter_contact": "contact@example.com", - "system_name": "system_j", - "system_category": "datacenter", - "max_supported_concurrency": 16384, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", - "host_processors_per_node": 36, - "host_processor_core_count": 72, - "host_memory_capacity": "2.16 TB LPDDR5x", - "host_networking": "InfiniBand NDR400", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04 (aarch64)", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA GB300", - "accelerators_per_node": 72, - "accelerator_memory_capacity": "288 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 5.0" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 72, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.067985 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/config.yaml deleted file mode 100644 index 74340a912..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 16384 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/point.yaml deleted file mode 100644 index 0c84640b6..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/point.yaml +++ /dev/null @@ -1,50 +0,0 @@ -concurrency: 16384 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 16384 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -offline: elected -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 16384 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 163840 - requests_issued: 163840 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/result_summary.json deleted file mode 100644 index 4a036b18c..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 417284, - "n_samples_completed": 417284, - "n_samples_failed": 0, - "duration_ns": 4165777685634.0, - "ttft": { - "total": 178378755811144.0, - "percentiles": { - "50": 427475666.0, - "95": 8529496162.349946, - "90": 6909092063.079957 - } - }, - "output_sequence_lengths": { - "total": 1562330261.0, - "percentiles": {} - }, - "tpot": { - "total": 1689582916000.0002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 375039.2793, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/system_desc.json deleted file mode 100644 index a47d5edd3..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r16384/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-J", - "submitter_contact": "contact@example.com", - "system_name": "system_j", - "system_category": "datacenter", - "max_supported_concurrency": 16384, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", - "host_processors_per_node": 36, - "host_processor_core_count": 72, - "host_memory_capacity": "2.16 TB LPDDR5x", - "host_networking": "InfiniBand NDR400", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04 (aarch64)", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA GB300", - "accelerators_per_node": 72, - "accelerator_memory_capacity": "288 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 5.0" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 72, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 1.0 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/config.yaml deleted file mode 100644 index 6b53d1bba..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 2048 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/point.yaml deleted file mode 100644 index 843d67b73..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 2048 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 16384 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 2048 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 20480 - requests_issued: 20480 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/result_summary.json deleted file mode 100644 index dabf2927b..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 8776, - "n_samples_completed": 8776, - "n_samples_failed": 0, - "duration_ns": 247915387632.0, - "ttft": { - "total": 4262954461920.0, - "percentiles": { - "50": 485751420.0, - "95": 3795499436.5, - "90": 3133549833.2000003 - } - }, - "output_sequence_lengths": { - "total": 33975939.0, - "percentiles": {} - }, - "tpot": { - "total": 35534024000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 137046.5114, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/system_desc.json deleted file mode 100644 index 0cf7c26d4..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r2048/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-J", - "submitter_contact": "contact@example.com", - "system_name": "system_j", - "system_category": "datacenter", - "max_supported_concurrency": 16384, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", - "host_processors_per_node": 36, - "host_processor_core_count": 72, - "host_memory_capacity": "2.16 TB LPDDR5x", - "host_networking": "InfiniBand NDR400", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04 (aarch64)", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA GB300", - "accelerators_per_node": 72, - "accelerator_memory_capacity": "288 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 5.0" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 72, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.365419 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/config.yaml deleted file mode 100644 index f928653b5..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 256 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/point.yaml deleted file mode 100644 index 83661b237..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 256 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 16384 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 256 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 2560 - requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/result_summary.json deleted file mode 100644 index 7b8c8f2b2..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 8776, - "n_samples_completed": 8776, - "n_samples_failed": 0, - "duration_ns": 782307149059.0001, - "ttft": { - "total": 2172863837720.0, - "percentiles": { - "50": 247591595.0, - "95": 505510358.75, - "90": 453926606.0 - } - }, - "output_sequence_lengths": { - "total": 33878035.0, - "percentiles": {} - }, - "tpot": { - "total": 35534024000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 43305.2862, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/system_desc.json deleted file mode 100644 index 136c33fac..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r256/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-J", - "submitter_contact": "contact@example.com", - "system_name": "system_j", - "system_category": "datacenter", - "max_supported_concurrency": 16384, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", - "host_processors_per_node": 36, - "host_processor_core_count": 72, - "host_memory_capacity": "2.16 TB LPDDR5x", - "host_networking": "InfiniBand NDR400", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04 (aarch64)", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA GB300", - "accelerators_per_node": 72, - "accelerator_memory_capacity": "288 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 5.0" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 72, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.115469 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/config.yaml deleted file mode 100644 index b03f20973..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 32 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/point.yaml deleted file mode 100644 index 54d1f355f..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 32 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 16384 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 32 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 320 - requests_issued: 320 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/result_summary.json deleted file mode 100644 index 2bfc7a8d6..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 8776, - "n_samples_completed": 8776, - "n_samples_failed": 0, - "duration_ns": 5076457317773.0, - "ttft": { - "total": 2109229364492.0, - "percentiles": { - "50": 240340629.5, - "95": 315733786.5, - "90": 300655155.1 - } - }, - "output_sequence_lengths": { - "total": 33926283.0, - "percentiles": {} - }, - "tpot": { - "total": 35534024000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 6683.0628, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/system_desc.json deleted file mode 100644 index d14e6a1df..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r32/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-J", - "submitter_contact": "contact@example.com", - "system_name": "system_j", - "system_category": "datacenter", - "max_supported_concurrency": 16384, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", - "host_processors_per_node": 36, - "host_processor_core_count": 72, - "host_memory_capacity": "2.16 TB LPDDR5x", - "host_networking": "InfiniBand NDR400", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04 (aarch64)", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA GB300", - "accelerators_per_node": 72, - "accelerator_memory_capacity": "288 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 5.0" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 72, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.01782 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/config.yaml deleted file mode 100644 index d3e2cf53c..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 4096 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/point.yaml deleted file mode 100644 index 136e3273f..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 4096 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 16384 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 4096 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 40960 - requests_issued: 40960 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/result_summary.json deleted file mode 100644 index f066438ca..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 8776, - "n_samples_completed": 8776, - "n_samples_failed": 0, - "duration_ns": 196988273307.0, - "ttft": { - "total": 45268755452096.0, - "percentiles": { - "50": 5158244696.0, - "95": 6765218642.5, - "90": 6443823853.2 - } - }, - "output_sequence_lengths": { - "total": 34013009.0, - "percentiles": {} - }, - "tpot": { - "total": 35534024000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 172665.1461, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/system_desc.json deleted file mode 100644 index 4b9e2c60f..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r4096/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-J", - "submitter_contact": "contact@example.com", - "system_name": "system_j", - "system_category": "datacenter", - "max_supported_concurrency": 16384, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", - "host_processors_per_node": 36, - "host_processor_core_count": 72, - "host_memory_capacity": "2.16 TB LPDDR5x", - "host_networking": "InfiniBand NDR400", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04 (aarch64)", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA GB300", - "accelerators_per_node": 72, - "accelerator_memory_capacity": "288 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 5.0" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 72, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.460392 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/config.yaml deleted file mode 100644 index 846832e5e..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 512 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/point.yaml deleted file mode 100644 index da7ab6472..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 512 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 16384 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 512 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 5120 - requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/result_summary.json deleted file mode 100644 index 65c5ee05d..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 8776, - "n_samples_completed": 8776, - "n_samples_failed": 0, - "duration_ns": 495036806948.0, - "ttft": { - "total": 2712508796676.0, - "percentiles": { - "50": 309082588.5, - "95": 569378711.0, - "90": 517319486.5 - } - }, - "output_sequence_lengths": { - "total": 34233018.0, - "percentiles": {} - }, - "tpot": { - "total": 35534024000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 69152.4701, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/system_desc.json deleted file mode 100644 index 48214dae4..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r512/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-J", - "submitter_contact": "contact@example.com", - "system_name": "system_j", - "system_category": "datacenter", - "max_supported_concurrency": 16384, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", - "host_processors_per_node": 36, - "host_processor_core_count": 72, - "host_memory_capacity": "2.16 TB LPDDR5x", - "host_networking": "InfiniBand NDR400", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04 (aarch64)", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA GB300", - "accelerators_per_node": 72, - "accelerator_memory_capacity": "288 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 5.0" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 72, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.184387 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/config.yaml deleted file mode 100644 index 7db6216f5..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 64 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/point.yaml deleted file mode 100644 index 6327faa28..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 64 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 16384 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 64 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 640 - requests_issued: 640 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/result_summary.json deleted file mode 100644 index 051a5ab73..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 8776, - "n_samples_completed": 8776, - "n_samples_failed": 0, - "duration_ns": 2602571018770.0, - "ttft": { - "total": 2027031584516.0, - "percentiles": { - "50": 230974428.5, - "95": 307411728.5, - "90": 292124268.5 - } - }, - "output_sequence_lengths": { - "total": 34185331.0, - "percentiles": {} - }, - "tpot": { - "total": 35534024000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 13135.2154, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/system_desc.json deleted file mode 100644 index ce9b3c3ed..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r64/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-J", - "submitter_contact": "contact@example.com", - "system_name": "system_j", - "system_category": "datacenter", - "max_supported_concurrency": 16384, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", - "host_processors_per_node": 36, - "host_processor_core_count": 72, - "host_memory_capacity": "2.16 TB LPDDR5x", - "host_networking": "InfiniBand NDR400", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04 (aarch64)", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA GB300", - "accelerators_per_node": 72, - "accelerator_memory_capacity": "288 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 5.0" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 72, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.035024 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/accuracy_results.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/accuracy_results.json deleted file mode 100644 index 0a70c3eb7..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/accuracy_results.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "math500": { - "dataset_name": "math500", - "num_samples": 4388, - "score": { - "exact_match": 81.3582 - }, - "extractor": "BoxedAnswerExtractor", - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/config.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/config.yaml deleted file mode 100644 index 69f442619..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 8192 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/point.yaml b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/point.yaml deleted file mode 100644 index 98f6e7483..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 8192 -dataset: mlperf-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 16384 -model_name: deepseek-r1 -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 8192 - data_source: mlperf-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 81920 - requests_issued: 81920 diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/result_summary.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/result_summary.json deleted file mode 100644 index 7d989d711..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 417284, - "n_samples_completed": 417284, - "n_samples_failed": 0, - "duration_ns": 6231386322474.0, - "ttft": { - "total": 165929879676176.0, - "percentiles": { - "50": 397642564.0, - "95": 641731587.85, - "90": 592913783.08 - } - }, - "output_sequence_lengths": { - "total": 1564302823.0, - "percentiles": {} - }, - "tpot": { - "total": 1689582916000.0002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 251036.0844, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/system_desc.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/system_desc.json deleted file mode 100644 index fdaec79c5..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/deepseek-r1/r8192/system_desc.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "submitter_org_names": "Submitter-J", - "submitter_contact": "contact@example.com", - "system_name": "system_j", - "system_category": "datacenter", - "max_supported_concurrency": 16384, - "serving_framework": "TensorRT-LLM", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "NVIDIA Grace CPU (Armv9)", - "host_processors_per_node": 36, - "host_processor_core_count": 72, - "host_memory_capacity": "2.16 TB LPDDR5x", - "host_networking": "InfiniBand NDR400", - "host_storage_capacity": "30 TB", - "host_storage_type": "NVMe SSD", - "operating_system": "Ubuntu 22.04 (aarch64)", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA GB300", - "accelerators_per_node": 72, - "accelerator_memory_capacity": "288 GB HBM3e", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 5.0" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "deepseek-r1", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 72, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "cooling": "Liquid-cooled", - "tps_utilization": 0.669359 -} diff --git a/tests/fixtures/validation/submissions/sub_j/results/system_j/system_power.json b/tests/fixtures/validation/submissions/sub_j/results/system_j/system_power.json deleted file mode 100644 index 142b43500..000000000 --- a/tests/fixtures/validation/submissions/sub_j/results/system_j/system_power.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "system_desc_id": "system_j", - "cooling": "liquid", - "node_sets": [ - { - "node_set_id": 0, - "system_node_ensemble_id": 0, - "nodes_provisioned": 1, - "power_method": "component_sum", - "components": { - "cpu": { - "model": "NVIDIA Grace CPU (Armv9)", - "count_per_node": 36, - "tdp_per_unit": { - "value_w": 350, - "source_type": "vendor_spec", - "source": "https://example.com/cpu-spec" - } - }, - "accelerator": { - "model": "NVIDIA GB300", - "count_per_node": 72, - "tdp_per_unit": { - "value_w": 700, - "source_type": "vendor_spec", - "source": "https://example.com/accelerator-spec" - } - }, - "scale_up_network": { - "method": "declared_tdp", - "switch_count": 1, - "tdp_per_switch": { - "value_w": 3500, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - } - } - ], - "scale_out": { - "present": false - } -} diff --git a/tests/fixtures/validation/submissions/sub_j/src/tensorrt-llm/README.md b/tests/fixtures/validation/submissions/sub_j/src/tensorrt-llm/README.md deleted file mode 100644 index ab1543fd0..000000000 --- a/tests/fixtures/validation/submissions/sub_j/src/tensorrt-llm/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# tensorrt-llm - -Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/fixtures/validation/submissions/valid_standardized/docs/.gitkeep b/tests/fixtures/validation/submissions/valid_standardized/docs/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/valid_standardized/docs/calibration.adoc b/tests/fixtures/validation/submissions/valid_standardized/docs/calibration.adoc deleted file mode 100644 index cc918cad0..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/docs/calibration.adoc +++ /dev/null @@ -1,5 +0,0 @@ -= Weight Transformation Documentation - -== Quantization - -Applied INT8 post-training quantization using GPTQ with 512-sample calibration dataset drawn from the performance dataset. No gradient updates applied. Calibration dataset disclosed in full in calibration_data/. diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/accuracy_results.json deleted file mode 100644 index b37d348d9..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/accuracy_results.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "llm-perf-dataset-v1": { - "dataset_name": "llm-perf-dataset-v1", - "num_samples": 13368, - "extractor": "RougeExtractor", - "score": { - "rouge1": "45.12", - "rouge2": "22.01", - "rougeL": "30.45" - }, - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/config.yaml deleted file mode 100644 index ad1450850..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 1000 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/point.yaml deleted file mode 100644 index 7085286ce..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/point.yaml +++ /dev/null @@ -1,50 +0,0 @@ -concurrency: 1000 -dataset: llm-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1000 -model_name: llama3_1-8b -model_notes: "" -model_precision: FP16 -offline: elected -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 1000 - data_source: llm-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 10000 - requests_issued: 10000 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/result_summary.json deleted file mode 100644 index 793001ed3..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 22000, - "n_samples_completed": 22000, - "n_samples_failed": 0, - "duration_ns": 1200000000000, - "ttft": { - "total": 9259800000000.0, - "percentiles": { - "50": 420900000.0, - "95": 890500000.0, - "90": 796580000.0 - } - }, - "output_sequence_lengths": { - "total": 5400000.0, - "percentiles": {} - }, - "tpot": { - "total": 89078000000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 4500.0, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/system_desc.json deleted file mode 100644 index b0dc4f6e2..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r1000/system_desc.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "submitter_org_names": "ACME Corp", - "submitter_contact": "contact@example.com", - "system_name": "ACME H100x8 Node", - "system_category": "datacenter", - "max_supported_concurrency": 1000, - "serving_framework": "vLLM 0.4.2", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "20 TB", - "host_storage_type": "NVMe SSD", - "cooling": "Air-cooled", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100 SXM5 80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3.1-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "model_name": "llama3.1-8b", - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "tps_utilization": 0.978261 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/accuracy_results.json deleted file mode 100644 index b37d348d9..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/accuracy_results.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "llm-perf-dataset-v1": { - "dataset_name": "llm-perf-dataset-v1", - "num_samples": 13368, - "extractor": "RougeExtractor", - "score": { - "rouge1": "45.12", - "rouge2": "22.01", - "rougeL": "30.45" - }, - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/config.yaml deleted file mode 100644 index f9afe658a..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 16 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/point.yaml deleted file mode 100644 index e19f1fe77..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 16 -dataset: llm-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1000 -model_name: llama3_1-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 600000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 600.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 16 - data_source: llm-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 160 - requests_issued: 160 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/result_summary.json deleted file mode 100644 index e6e2bc2a1..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 3000, - "n_samples_completed": 3000, - "n_samples_failed": 0, - "duration_ns": 600000000000, - "ttft": { - "total": 135600000000.0, - "percentiles": { - "50": 45200000.0, - "95": 89100000.0, - "90": 80320000.0 - } - }, - "output_sequence_lengths": { - "total": 480000.0, - "percentiles": {} - }, - "tpot": { - "total": 12147000000.000002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 800.0, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/system_desc.json deleted file mode 100644 index cfa21d246..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/system_desc.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "submitter_org_names": "ACME Corp", - "submitter_contact": "contact@example.com", - "system_name": "ACME H100x8 Node", - "system_category": "datacenter", - "max_supported_concurrency": 1000, - "serving_framework": "vLLM 0.4.2", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "20 TB", - "host_storage_type": "NVMe SSD", - "cooling": "Air-cooled", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100 SXM5 80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3.1-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "model_name": "llama3.1-8b", - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "tps_utilization": 0.173913 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/accuracy_results.json deleted file mode 100644 index b37d348d9..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/accuracy_results.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "llm-perf-dataset-v1": { - "dataset_name": "llm-perf-dataset-v1", - "num_samples": 13368, - "extractor": "RougeExtractor", - "score": { - "rouge1": "45.12", - "rouge2": "22.01", - "rougeL": "30.45" - }, - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/config.yaml deleted file mode 100644 index 1a6f2cb8e..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 38 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/point.yaml deleted file mode 100644 index 39fd6248b..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 20 -dataset: llm-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1000 -model_name: llama3_1-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 20 - data_source: llm-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 200 - requests_issued: 200 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/result_summary.json deleted file mode 100644 index 28f9c21c7..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 8000, - "n_samples_completed": 8000, - "n_samples_failed": 0, - "duration_ns": 1200000000000, - "ttft": { - "total": 418400000000.0, - "percentiles": { - "50": 52300000.0, - "95": 110400000.0, - "90": 98780000.0 - } - }, - "output_sequence_lengths": { - "total": 1800000.0, - "percentiles": {} - }, - "tpot": { - "total": 32392000000.000004, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 1500.0, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/system_desc.json deleted file mode 100644 index 54c2146a6..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r20/system_desc.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "submitter_org_names": "ACME Corp", - "submitter_contact": "contact@example.com", - "system_name": "ACME H100x8 Node", - "system_category": "datacenter", - "max_supported_concurrency": 1000, - "serving_framework": "vLLM 0.4.2", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "20 TB", - "host_storage_type": "NVMe SSD", - "cooling": "Air-cooled", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100 SXM5 80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3.1-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "model_name": "llama3.1-8b", - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "tps_utilization": 0.326087 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/accuracy_results.json deleted file mode 100644 index b37d348d9..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/accuracy_results.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "llm-perf-dataset-v1": { - "dataset_name": "llm-perf-dataset-v1", - "num_samples": 13368, - "extractor": "RougeExtractor", - "score": { - "rouge1": "45.12", - "rouge2": "22.01", - "rougeL": "30.45" - }, - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/config.yaml deleted file mode 100644 index f928653b5..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 256 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/point.yaml deleted file mode 100644 index d85e69bae..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 256 -dataset: llm-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1000 -model_name: llama3_1-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 256 - data_source: llm-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 2560 - requests_issued: 2560 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/result_summary.json deleted file mode 100644 index b9aa245f8..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 20000, - "n_samples_completed": 20000, - "n_samples_failed": 0, - "duration_ns": 1200000000000, - "ttft": { - "total": 2050000000000.0, - "percentiles": { - "50": 102500000.0, - "95": 220800000.0, - "90": 197140000.0 - } - }, - "output_sequence_lengths": { - "total": 4800000.0, - "percentiles": {} - }, - "tpot": { - "total": 80980000000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 4000.0, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/system_desc.json deleted file mode 100644 index 01ab8325f..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r256/system_desc.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "submitter_org_names": "ACME Corp", - "submitter_contact": "contact@example.com", - "system_name": "ACME H100x8 Node", - "system_category": "datacenter", - "max_supported_concurrency": 1000, - "serving_framework": "vLLM 0.4.2", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "20 TB", - "host_storage_type": "NVMe SSD", - "cooling": "Air-cooled", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100 SXM5 80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3.1-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "model_name": "llama3.1-8b", - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "tps_utilization": 0.869565 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/accuracy_results.json deleted file mode 100644 index b37d348d9..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/accuracy_results.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "llm-perf-dataset-v1": { - "dataset_name": "llm-perf-dataset-v1", - "num_samples": 13368, - "extractor": "RougeExtractor", - "score": { - "rouge1": "45.12", - "rouge2": "22.01", - "rougeL": "30.45" - }, - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/config.yaml deleted file mode 100644 index 846832e5e..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 512 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/point.yaml deleted file mode 100644 index f6c0decb8..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 512 -dataset: llm-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1000 -model_name: llama3_1-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 512 - data_source: llm-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 5120 - requests_issued: 5120 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/result_summary.json deleted file mode 100644 index 5c94bee96..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 22000, - "n_samples_completed": 22000, - "n_samples_failed": 0, - "duration_ns": 1200000000000, - "ttft": { - "total": 3966600000000.0, - "percentiles": { - "50": 180300000.0, - "95": 410200000.0, - "90": 364220000.0 - } - }, - "output_sequence_lengths": { - "total": 5400000.0, - "percentiles": {} - }, - "tpot": { - "total": 89078000000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 4500.0, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/system_desc.json deleted file mode 100644 index b0dc4f6e2..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r512/system_desc.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "submitter_org_names": "ACME Corp", - "submitter_contact": "contact@example.com", - "system_name": "ACME H100x8 Node", - "system_category": "datacenter", - "max_supported_concurrency": 1000, - "serving_framework": "vLLM 0.4.2", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "20 TB", - "host_storage_type": "NVMe SSD", - "cooling": "Air-cooled", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100 SXM5 80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3.1-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "model_name": "llama3.1-8b", - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "tps_utilization": 0.978261 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/accuracy_results.json deleted file mode 100644 index b37d348d9..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/accuracy_results.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "llm-perf-dataset-v1": { - "dataset_name": "llm-perf-dataset-v1", - "num_samples": 13368, - "extractor": "RougeExtractor", - "score": { - "rouge1": "45.12", - "rouge2": "22.01", - "rougeL": "30.45" - }, - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/config.yaml deleted file mode 100644 index 240492592..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 768 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/point.yaml deleted file mode 100644 index c120f6e32..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 768 -dataset: llm-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1000 -model_name: llama3_1-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 768 - data_source: llm-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 7680 - requests_issued: 7680 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/result_summary.json deleted file mode 100644 index c44d18beb..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 23000, - "n_samples_completed": 23000, - "n_samples_failed": 0, - "duration_ns": 1200000000000, - "ttft": { - "total": 6801100000000.0, - "percentiles": { - "50": 295700000.0, - "95": 620100000.0, - "90": 555220000.0 - } - }, - "output_sequence_lengths": { - "total": 5520000.0, - "percentiles": {} - }, - "tpot": { - "total": 93127000000.00002, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 4600.0, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/system_desc.json deleted file mode 100644 index 8a9e0fd70..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r768/system_desc.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "submitter_org_names": "ACME Corp", - "submitter_contact": "contact@example.com", - "system_name": "ACME H100x8 Node", - "system_category": "datacenter", - "max_supported_concurrency": 1000, - "serving_framework": "vLLM 0.4.2", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "20 TB", - "host_storage_type": "NVMe SSD", - "cooling": "Air-cooled", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100 SXM5 80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3.1-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "model_name": "llama3.1-8b", - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "tps_utilization": 1.0 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/accuracy_results.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/accuracy_results.json deleted file mode 100644 index b37d348d9..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/accuracy_results.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "llm-perf-dataset-v1": { - "dataset_name": "llm-perf-dataset-v1", - "num_samples": 13368, - "extractor": "RougeExtractor", - "score": { - "rouge1": "45.12", - "rouge2": "22.01", - "rougeL": "30.45" - }, - "n_repeats": 1 - } -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/config.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/config.yaml deleted file mode 100644 index cf16abd9b..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/config.yaml +++ /dev/null @@ -1 +0,0 @@ -concurrency: 88 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/point.yaml b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/point.yaml deleted file mode 100644 index 34bb7c7fc..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/point.yaml +++ /dev/null @@ -1,49 +0,0 @@ -concurrency: 88 -dataset: llm-perf-dataset-v1 -dataset_link: https://example.com/dataset -dataset_name: CNN/DailyMail -dataset_type: Performance -division: Standardized -link_to_model: https://example.com/model -link_to_model_transformation: https://example.com/quantization -max_supported_concurrency: 1000 -model_name: llama3_1-8b -model_notes: "" -model_precision: FP16 -runtime_settings: - load_pattern: concurrency - min_duration_ms: 1200000 - runtime: - model_seed: 9315206023656308754 - sample_index_rng_seed: 586478644936801402 - scheduler_rng_seed: 10487924139932647040 - stream_all_chunks: true -seed_set: A -shared_docs: docs -shared_src: src -steady_state: - state: - tpot_p50: Plateau - tpot_p90: Plateau - ttft_p50: Plateau - ttft_p90: Plateau - status: windowable - verdict: STEADY STATE - window: - duration_s: 1200.0 - n_samples: 4000 - super_pass_end: 4 - super_pass_size: 1000 - super_pass_start: 1 -target_cohort: 2026-10-C1 -warmup: - concurrency: 88 - data_source: llm-perf-dataset-v1 validation split - duration_s: 60.0 - initialization_steps: - - model loaded - - kv-cache warmed - link_logs: https://example.com/warmup-logs - logs_retained: true - requests_completed: 880 - requests_issued: 880 diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/result_summary.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/result_summary.json deleted file mode 100644 index aee0f8a03..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/result_summary.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "n_samples_issued": 12000, - "n_samples_completed": 12000, - "n_samples_failed": 0, - "duration_ns": 1200000000000, - "ttft": { - "total": 817200000000.0, - "percentiles": { - "50": 68100000.0, - "95": 145200000.0, - "90": 129780000.0 - } - }, - "output_sequence_lengths": { - "total": 3000000.0, - "percentiles": {} - }, - "tpot": { - "total": 48588000000.00001, - "percentiles": { - "50": 4000000.0, - "90": 5500000.0, - "95": 6000000.0, - "99": 6500000.0 - } - }, - "system_tps": 2500.0, - "tps_per_user": 181.8182 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/system_desc.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/system_desc.json deleted file mode 100644 index 1cfedfcba..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r88/system_desc.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "submitter_org_names": "ACME Corp", - "submitter_contact": "contact@example.com", - "system_name": "ACME H100x8 Node", - "system_category": "datacenter", - "max_supported_concurrency": 1000, - "serving_framework": "vLLM 0.4.2", - "node_types": [ - { - "system_node_ensemble_id": 0, - "number_of_nodes": 1, - "host_processor_model_name": "AMD EPYC 9654", - "host_processors_per_node": 2, - "host_processor_core_count": 96, - "host_memory_capacity": "768 GB", - "host_networking": "InfiniBand HDR", - "host_storage_capacity": "20 TB", - "host_storage_type": "NVMe SSD", - "cooling": "Air-cooled", - "operating_system": "Ubuntu 22.04", - "host_memory_configuration": "8x 64GB DDR5", - "host_network_card_count": "1x NIC", - "driver": "1.0", - "filesystem": "ext4", - "accelerator_info": [ - { - "accelerator_model_name": "NVIDIA H100 SXM5 80GB", - "accelerators_per_node": 8, - "accelerator_memory_capacity": "80 GB HBM3", - "accelerator_memory_type": "HBM3e", - "accelerator_interconnect": "NVLink 4.0 900 GB/s", - "accelerator_host_interconnect": "PCIe Gen5" - } - ] - } - ], - "division": "Standardized", - "model_precision": "FP16", - "link_to_model": "https://example.com/model", - "dataset_id": "cnn_dailymail", - "dataset_name": "CNN/DailyMail", - "input_token_average": 870.0, - "output_token_average": 128.0, - "dataset_type": "performance", - "dataset_link": "https://example.com/dataset", - "measured_accuracy_score": "38.7", - "system_size": "1 node", - "model_id": "llama3.1-8b", - "system_node_ensemble_count": 1, - "system_node_ensemble_total": 1, - "model_name": "llama3.1-8b", - "publication_status": "Available", - "node_config": "8x H100 test node", - "config_summary": { - "disaggregated": null, - "expert_parallel": 1, - "tensor_parallel": 8, - "pipeline_parallel": 1, - "data_parallel": 1, - "batch": null - }, - "tps_utilization": 0.543478 -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/system_power.json b/tests/fixtures/validation/submissions/valid_standardized/results/system_k/system_power.json deleted file mode 100644 index 1d5e9a43e..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/results/system_k/system_power.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "system_desc_id": "system_k", - "cooling": "air", - "node_sets": [ - { - "node_set_id": 0, - "system_node_ensemble_id": 0, - "nodes_provisioned": 1, - "power_method": "component_sum", - "components": { - "cpu": { - "model": "AMD EPYC 9654", - "count_per_node": 2, - "tdp_per_unit": { - "value_w": 350, - "source_type": "vendor_spec", - "source": "https://example.com/cpu-spec" - } - }, - "accelerator": { - "model": "NVIDIA H100 SXM5 80GB", - "count_per_node": 8, - "tdp_per_unit": { - "value_w": 700, - "source_type": "vendor_spec", - "source": "https://example.com/accelerator-spec" - } - }, - "scale_up_network": { - "method": "declared_tdp", - "switch_count": 1, - "tdp_per_switch": { - "value_w": 3500, - "source_type": "vendor_spec", - "source": "https://example.com/switch-spec" - } - } - } - } - ], - "scale_out": { - "present": false - } -} diff --git a/tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/.gitkeep b/tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/README.md b/tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/README.md deleted file mode 100644 index 45692c1a3..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/src/llama3_1-8b/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# llama3_1-8b - -How to build the SUT and reproduce a Pareto point. diff --git a/tests/fixtures/validation/submissions/valid_standardized/src/vllm/README.md b/tests/fixtures/validation/submissions/valid_standardized/src/vllm/README.md deleted file mode 100644 index be3edfb30..000000000 --- a/tests/fixtures/validation/submissions/valid_standardized/src/vllm/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# vllm - -Anonymised fixture implementation directory. Real submissions document how to build the SUT and reproduce a point here. diff --git a/tests/unit/validation/conftest.py b/tests/unit/validation/conftest.py new file mode 100644 index 000000000..a62479f59 --- /dev/null +++ b/tests/unit/validation/conftest.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Materialize readable submission cases as complete artifact trees.""" + +import json +from copy import deepcopy +from pathlib import Path + +import pytest +import yaml +from pydantic import JsonValue + +CASE_DIRECTORY = Path(__file__).resolve().parents[2] / "fixtures/validation/cases" +SUBMISSION_CASES = tuple(path.stem for path in sorted(CASE_DIRECTORY.glob("*.yaml"))) + + +def _merge( + base: dict[str, JsonValue], changes: dict[str, JsonValue] +) -> dict[str, JsonValue]: + merged = deepcopy(base) + for key, value in changes.items(): + existing = merged.get(key) + merged[key] = ( + _merge(existing, value) + if isinstance(existing, dict) and isinstance(value, dict) + else deepcopy(value) + ) + return merged + + +def _write_artifact(path: Path, value: JsonValue) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if path.suffix == ".json": + text = json.dumps(value, indent=2) + "\n" + elif path.suffix == ".yaml": + text = yaml.safe_dump(value, sort_keys=False) + else: + assert isinstance(value, str) + text = value + path.write_text(text, encoding="utf-8") + + +@pytest.fixture(scope="session") +def submission_corpus(tmp_path_factory: pytest.TempPathFactory) -> Path: + root = tmp_path_factory.mktemp("validation-submissions") + for name in SUBMISSION_CASES: + case = yaml.safe_load( + (CASE_DIRECTORY / f"{name}.yaml").read_text(encoding="utf-8") + ) + target = root / name + for filename, value in case["files"].items(): + _write_artifact(target / filename, value) + for point, files in case["points"].items(): + for filename, overrides in files.items(): + value = _merge(case["defaults"][filename], overrides) + _write_artifact(target / point / filename, value) + return root + + +@pytest.fixture(scope="session") +def standardized_submission(submission_corpus: Path) -> Path: + return submission_corpus / "valid_standardized" + + +@pytest.fixture(params=SUBMISSION_CASES, ids=SUBMISSION_CASES) +def submission_case(request: pytest.FixtureRequest, submission_corpus: Path) -> Path: + return submission_corpus / request.param diff --git a/tests/unit/validation/test_engine.py b/tests/unit/validation/test_engine.py index ec6627d8d..2e109bab4 100644 --- a/tests/unit/validation/test_engine.py +++ b/tests/unit/validation/test_engine.py @@ -5,7 +5,6 @@ import json import shutil from dataclasses import replace -from pathlib import Path import pytest import yaml @@ -25,10 +24,6 @@ from inference_endpoint.validation.vocabulary import CheckKind pytestmark = pytest.mark.unit -FIXTURE = ( - Path(__file__).resolve().parents[2] - / "fixtures/validation/submissions/valid_standardized" -) def policy_only(*ids): @@ -38,22 +33,24 @@ def policy_only(*ids): ) -def test_min_completed_samples_uses_catalog_sample_count(): +def test_min_completed_samples_uses_catalog_sample_count(standardized_submission): policy = policy_only("min-completed-samples") - original = execute_submission(FIXTURE, policy) + original = execute_submission(standardized_submission, policy) catalog = { **policy.catalogs, "datasets": {"llm-perf-dataset-v1": {"sample_count": 10**12}}, } - stricter = execute_submission(FIXTURE, replace(policy, catalogs=freeze(catalog))) + stricter = execute_submission( + standardized_submission, replace(policy, catalogs=freeze(catalog)) + ) assert original.passed assert not stricter.passed assert all(f.rule == "min-completed-samples" for f in stricter.results) -def test_duration_catalog_edit_changes_runtime_gate(): +def test_duration_catalog_edit_changes_runtime_gate(standardized_submission): policy = policy_only("region-basis", "point-duration") - original = execute_submission(FIXTURE, policy) + original = execute_submission(standardized_submission, policy) catalog = { **policy.catalogs, "steady_state": { @@ -62,12 +59,16 @@ def test_duration_catalog_edit_changes_runtime_gate(): ) }, } - stricter = execute_submission(FIXTURE, replace(policy, catalogs=freeze(catalog))) + stricter = execute_submission( + standardized_submission, replace(policy, catalogs=freeze(catalog)) + ) assert original.passed assert not stricter.passed -def test_new_yaml_rule_and_selection_override_execute(tmp_path): +def test_new_yaml_rule_and_selection_override_execute( + standardized_submission, tmp_path +): policy_dir = tmp_path / "2026-10-C1" shutil.copytree(bundled_policy_path(), policy_dir) point_yaml = policy_dir / "point_checks.yaml" @@ -83,19 +84,28 @@ def test_new_yaml_rule_and_selection_override_execute(tmp_path): point_yaml.write_text(yaml.safe_dump(document)) policy = load_policy(policy_dir) rule = next(r for r in policy.checks if r.id == "new-completed-floor") - assert not execute_submission(FIXTURE, replace(policy, checks=(rule,))).passed + assert not execute_submission( + standardized_submission, replace(policy, checks=(rule,)) + ).passed document["checks"]["new-completed-floor"]["applies_to"] = { "load_pattern": ["agentic_inference"] } point_yaml.write_text(yaml.safe_dump(document)) policy = load_policy(policy_dir) rule = next(r for r in policy.checks if r.id == "new-completed-floor") - assert execute_submission(FIXTURE, replace(policy, checks=(rule,))).results == [] + assert ( + execute_submission( + standardized_submission, replace(policy, checks=(rule,)) + ).results + == [] + ) -def test_untrusted_nan_region_basis_blocks_without_crashing(tmp_path): +def test_untrusted_nan_region_basis_blocks_without_crashing( + standardized_submission, tmp_path +): submission = tmp_path / "submission" - shutil.copytree(FIXTURE, submission) + shutil.copytree(standardized_submission, submission) for path in submission.glob("results/*/*/r*/system_desc.json"): data = json.loads(path.read_text()) data["max_supported_concurrency"] = float("nan") @@ -114,7 +124,7 @@ def __call__(self, check, artifacts): return [] -def test_model_accuracy_catalog_changes_public_results(): +def test_model_accuracy_catalog_changes_public_results(standardized_submission): policy = policy_only("accuracy-gate") profile = { "accuracy": { @@ -125,13 +135,13 @@ def test_model_accuracy_catalog_changes_public_results(): } catalogs = {**policy.catalogs, "models": {"llama3_1-8b": profile}} policy = replace(policy, catalogs=freeze(catalogs)) - assert validate_submission(FIXTURE, policy=policy).passed + assert validate_submission(standardized_submission, policy=policy).passed profile["accuracy"]["metrics"]["rouge1"]["reference"] = 10**12 stricter = replace(policy, catalogs=freeze(catalogs)) - assert not validate_submission(FIXTURE, policy=stricter).passed + assert not validate_submission(standardized_submission, policy=stricter).passed -def test_disabled_cohort_override_prevents_execution(tmp_path): +def test_disabled_cohort_override_prevents_execution(standardized_submission, tmp_path): policy_dir = tmp_path / "2026-10-C1" shutil.copytree(bundled_policy_path(), policy_dir) catalog_file = policy_dir / "catalog.yaml" @@ -153,17 +163,19 @@ def test_disabled_cohort_override_prevents_execution(tmp_path): rule for rule in policy.checks if rule.id == "min-completed-samples" ), ) - assert validate_submission(FIXTURE, policy=policy).results == [] + assert validate_submission(standardized_submission, policy=policy).results == [] -def test_unregistered_evaluator_blocks(monkeypatch): +def test_unregistered_evaluator_blocks(standardized_submission, monkeypatch): monkeypatch.delitem(Evaluator.registry, CheckKind.COMPARISON) - report = execute_submission(FIXTURE, policy_only("metric-consistency-duration")) + report = execute_submission( + standardized_submission, policy_only("metric-consistency-duration") + ) assert report.errors assert all(f.key == "blocked" for f in report.errors) -def test_effective_override_changes_evaluator_threshold(): +def test_effective_override_changes_evaluator_threshold(standardized_submission): policy = policy_only("point-cap") override = Override( id="smaller-cohort-cap", @@ -173,15 +185,15 @@ def test_effective_override_changes_evaluator_threshold(): enabled=True, requirements=freeze({"maximum": 1}), ) - assert validate_submission(FIXTURE, policy=policy).passed + assert validate_submission(standardized_submission, policy=policy).passed overridden = validate_submission( - FIXTURE, policy=replace(policy, overrides=(override,)) + standardized_submission, policy=replace(policy, overrides=(override,)) ) assert not overridden.passed assert all("allowed [0, 1]" in finding.message for finding in overridden.results) -def test_disabled_region_basis_does_not_publish_dependencies(): +def test_disabled_region_basis_does_not_publish_dependencies(standardized_submission): policy = policy_only("region-basis") exemption = Override( id="disabled-basis", @@ -191,7 +203,9 @@ def test_disabled_region_basis_does_not_publish_dependencies(): enabled=False, requirements=freeze({}), ) - artifacts = load_artifacts(FIXTURE, replace(policy, overrides=(exemption,))) + artifacts = load_artifacts( + standardized_submission, replace(policy, overrides=(exemption,)) + ) assert not artifacts.derived.regions assert all( EvidenceKey.COMPUTED_REGIONS not in point.context.available @@ -199,9 +213,11 @@ def test_disabled_region_basis_does_not_publish_dependencies(): ) -def test_region_basis_uses_selected_collection_members(tmp_path): +def test_region_basis_uses_selected_collection_members( + standardized_submission, tmp_path +): submission = tmp_path / "submission" - shutil.copytree(FIXTURE, submission) + shutil.copytree(standardized_submission, submission) path = next(submission.glob("results/*/*/r16/point.yaml")) data = yaml.safe_load(path.read_text()) data["offline"] = "dedicated" @@ -219,9 +235,11 @@ def test_unknown_collection_operation_is_rejected(rule_id): rule.requirements.with_updates({"operation": "unsupported"}) -def test_offline_uses_effective_numeric_concurrency_floor(tmp_path): +def test_offline_uses_effective_numeric_concurrency_floor( + standardized_submission, tmp_path +): submission = tmp_path / "submission" - shutil.copytree(FIXTURE, submission) + shutil.copytree(standardized_submission, submission) path = next(submission.glob("results/*/*/r1000/point.yaml")) data = yaml.safe_load(path.read_text()) data["offline"] = "dedicated" @@ -238,7 +256,7 @@ def test_offline_uses_effective_numeric_concurrency_floor(tmp_path): assert "1000000000000" in report.results[0].message -def test_elected_offline_uses_effective_required_concurrency(): +def test_elected_offline_uses_effective_required_concurrency(standardized_submission): policy = policy_only("offline-point-present") rule = replace( policy.checks[0], @@ -246,12 +264,16 @@ def test_elected_offline_uses_effective_required_concurrency(): {"elected_must_equal": 10**12} ), ) - assert not execute_submission(FIXTURE, replace(policy, checks=(rule,))).passed + assert not execute_submission( + standardized_submission, replace(policy, checks=(rule,)) + ).passed -def test_schema_errors_use_effective_warning_severity(tmp_path): +def test_schema_errors_use_effective_warning_severity( + standardized_submission, tmp_path +): submission = tmp_path / "submission" - shutil.copytree(FIXTURE, submission) + shutil.copytree(standardized_submission, submission) path = next(submission.glob("results/*/*/r16/point.yaml")) data = yaml.safe_load(path.read_text()) data["concurrency"] = "invalid" @@ -263,7 +285,7 @@ def test_schema_errors_use_effective_warning_severity(tmp_path): assert all(f.severity.value == "warning" for f in report.results if f.path == path) -def test_conflicting_region_basis_partitions_block_explicitly(): +def test_conflicting_region_basis_partitions_block_explicitly(standardized_submission): policy = policy_only("region-basis", "point-duration") partition = Override( id="different-clamp", @@ -273,7 +295,9 @@ def test_conflicting_region_basis_partitions_block_explicitly(): enabled=True, requirements=freeze({"upper_clamp": 4}), ) - report = execute_submission(FIXTURE, replace(policy, overrides=(partition,))) + report = execute_submission( + standardized_submission, replace(policy, overrides=(partition,)) + ) assert any( f.rule == "region-basis" and f.key == "blocked" and "Conflicting" in f.message for f in report.results @@ -286,9 +310,9 @@ def test_conflicting_region_basis_partitions_block_explicitly(): @pytest.mark.parametrize( "option", ["seed_sets_path", "approved_spec_decode_heads_path"] ) -def test_api_rejects_catalog_overrides(option, tmp_path): +def test_api_rejects_catalog_overrides(standardized_submission, option, tmp_path): with pytest.raises(TypeError, match="unexpected keyword argument"): - validate_submission(FIXTURE, **{option: tmp_path}) + validate_submission(standardized_submission, **{option: tmp_path}) def test_catalog_environment_cannot_replace_validation_approvals(tmp_path, monkeypatch): @@ -306,11 +330,8 @@ def test_catalog_environment_cannot_replace_validation_approvals(tmp_path, monke ) -@pytest.mark.parametrize( - "submission", sorted(FIXTURE.parent.iterdir()), ids=lambda path: path.name -) -def test_native_submission_corpus_has_no_internal_evaluation_errors(submission): - report = validate_submission(submission) +def test_native_submission_corpus_has_no_internal_evaluation_errors(submission_case): + report = validate_submission(submission_case) assert report.results assert not any( "Evaluation could not complete" in finding.message for finding in report.results diff --git a/tests/unit/validation/test_final_review_regressions.py b/tests/unit/validation/test_final_review_regressions.py index e3f0fb163..2e5d1b94b 100644 --- a/tests/unit/validation/test_final_review_regressions.py +++ b/tests/unit/validation/test_final_review_regressions.py @@ -5,7 +5,6 @@ import json import shutil from dataclasses import replace -from pathlib import Path import pytest import yaml @@ -29,10 +28,6 @@ ) pytestmark = pytest.mark.unit -FIXTURE = ( - Path(__file__).resolve().parents[2] - / "fixtures/validation/submissions/valid_standardized" -) def selected_policy(*ids): @@ -51,8 +46,10 @@ def write_summary(root, summary): return point -def test_system_names_are_compared_within_each_system(tmp_path): - shutil.copytree(FIXTURE, tmp_path / "submission") +def test_system_names_are_compared_within_each_system( + standardized_submission, tmp_path +): + shutil.copytree(standardized_submission, tmp_path / "submission") root = tmp_path / "submission" systems = root / "results" original = next(systems.iterdir()) diff --git a/tests/unit/validation/test_policy_contracts.py b/tests/unit/validation/test_policy_contracts.py index 506b7b88b..aaea13004 100644 --- a/tests/unit/validation/test_policy_contracts.py +++ b/tests/unit/validation/test_policy_contracts.py @@ -4,7 +4,6 @@ import json import shutil -from pathlib import Path import pytest import yaml @@ -296,12 +295,11 @@ def test_malformed_nested_and_operation_contracts_are_rejected( ], ) def test_power_cooling_must_match_description( - tmp_path, description, descriptor_cooling, accepted + standardized_submission, tmp_path, description, descriptor_cooling, accepted ): policy = load_policy(bundled_policy_path()) source = ( - Path(__file__).resolve().parents[3] - / "tests/fixtures/validation/submissions/valid_standardized/results/system_k/llama3_1-8b/r16/system_desc.json" + standardized_submission / "results/system_k/llama3_1-8b/r16/system_desc.json" ) system = tmp_path / "results" / "system" point = system / "gpt-oss-120b" / "r16" From 0a469ab67206bb5b8d5e9159123380972903d00e Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 14:00:32 -0700 Subject: [PATCH 16/18] test(validation): consolidate repeated setup and seed cases --- tests/unit/validation/conftest.py | 11 ++ tests/unit/validation/helpers.py | 31 ++++ tests/unit/validation/test_bindings.py | 13 +- tests/unit/validation/test_counts.py | 6 +- tests/unit/validation/test_engine.py | 51 ++---- .../test_final_review_regressions.py | 26 +-- tests/unit/validation/test_loader.py | 8 - tests/unit/validation/test_planner.py | 7 +- .../unit/validation/test_policy_contracts.py | 8 - tests/unit/validation/test_seed_catalog.py | 105 ++++------- .../unit/validation/test_spec_decode_heads.py | 168 +++++------------- .../validation/test_special_evaluators.py | 12 +- 12 files changed, 153 insertions(+), 293 deletions(-) create mode 100644 tests/unit/validation/helpers.py diff --git a/tests/unit/validation/conftest.py b/tests/unit/validation/conftest.py index a62479f59..9ed97cb2c 100644 --- a/tests/unit/validation/conftest.py +++ b/tests/unit/validation/conftest.py @@ -3,6 +3,7 @@ """Materialize readable submission cases as complete artifact trees.""" import json +import shutil from copy import deepcopy from pathlib import Path @@ -10,6 +11,8 @@ import yaml from pydantic import JsonValue +from inference_endpoint.validation import bundled_policy_path + CASE_DIRECTORY = Path(__file__).resolve().parents[2] / "fixtures/validation/cases" SUBMISSION_CASES = tuple(path.stem for path in sorted(CASE_DIRECTORY.glob("*.yaml"))) @@ -65,3 +68,11 @@ def standardized_submission(submission_corpus: Path) -> Path: @pytest.fixture(params=SUBMISSION_CASES, ids=SUBMISSION_CASES) def submission_case(request: pytest.FixtureRequest, submission_corpus: Path) -> Path: return submission_corpus / request.param + + +@pytest.fixture +def policy_dir(tmp_path: Path) -> Path: + """Give each policy mutation test an independent bundle.""" + target = tmp_path / "2026-10-C1" + shutil.copytree(bundled_policy_path(), target) + return target diff --git a/tests/unit/validation/helpers.py b/tests/unit/validation/helpers.py new file mode 100644 index 000000000..4bbfdbd6b --- /dev/null +++ b/tests/unit/validation/helpers.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared setup for focused validation tests.""" + +from dataclasses import replace + +from pydantic import JsonValue + +from inference_endpoint.validation import bundled_policy_path, load_policy +from inference_endpoint.validation.artifacts import PointArtifacts +from inference_endpoint.validation.evidence.loaders import ParsedArtifact +from inference_endpoint.validation.models import Policy + + +def selected_policy(*ids: str, policy: Policy | None = None) -> Policy: + policy = policy if policy is not None else load_policy(bundled_policy_path()) + return replace( + policy, checks=tuple(rule for rule in policy.checks if rule.id in ids) + ) + + +def update_evidence(point: PointArtifacts, field: str, **updates: JsonValue) -> None: + artifact = getattr(point.evidence, field) + assert artifact.value is not None + data = artifact.value.model_dump( + mode="json", exclude_unset=True, exclude_computed_fields=True + ) + data.update(updates) + parsed = ParsedArtifact.from_json(data, type(artifact.value), point.path, field) + assert not parsed.errors + point.evidence = replace(point.evidence, **{field: parsed}) diff --git a/tests/unit/validation/test_bindings.py b/tests/unit/validation/test_bindings.py index c6b2498e2..d621ae0be 100644 --- a/tests/unit/validation/test_bindings.py +++ b/tests/unit/validation/test_bindings.py @@ -14,25 +14,14 @@ ) from inference_endpoint.validation.bindings import ArtifactBinding from inference_endpoint.validation.conditions import Context -from inference_endpoint.validation.evidence.loaders import ParsedArtifact from inference_endpoint.validation.models import freeze from inference_endpoint.validation.planner import PlannedCheck from inference_endpoint.validation.types import Decision, Scope +from tests.unit.validation.helpers import update_evidence pytestmark = pytest.mark.unit -def update_evidence(point, field, **updates): - artifact = getattr(point.evidence, field) - data = artifact.value.model_dump( - mode="json", exclude_unset=True, exclude_computed_fields=True - ) - data.update(updates) - parsed = ParsedArtifact.from_json(data, type(artifact.value), point.path, field) - assert not parsed.errors - point.evidence = replace(point.evidence, **{field: parsed}) - - def binding_check(tmp_path, rule_id): policy = load_policy(bundled_policy_path()) context = Context(id=str(tmp_path), scope=Scope.POINT, model_id="kimi-k3") diff --git a/tests/unit/validation/test_counts.py b/tests/unit/validation/test_counts.py index 9c9cd9ded..d800a2f98 100644 --- a/tests/unit/validation/test_counts.py +++ b/tests/unit/validation/test_counts.py @@ -3,7 +3,6 @@ """Reported and configured counts share operators while retaining rule semantics.""" import json -import shutil from dataclasses import replace import pytest @@ -150,9 +149,8 @@ def test_fixed_reference_count_missing_blocks_check(tmp_path, policy, rule_id): }, ], ) -def test_count_requirements_reject_ambiguous_or_invalid_bindings(tmp_path, changes): - target = tmp_path / "2026-10-C1" - shutil.copytree(bundled_policy_path(), target) +def test_count_requirements_reject_ambiguous_or_invalid_bindings(policy_dir, changes): + target = policy_dir path = target / "point_checks.yaml" document = yaml.safe_load(path.read_text()) document["checks"]["trajectories-to-issue"].update(changes) diff --git a/tests/unit/validation/test_engine.py b/tests/unit/validation/test_engine.py index 2e109bab4..df61d7bb3 100644 --- a/tests/unit/validation/test_engine.py +++ b/tests/unit/validation/test_engine.py @@ -22,19 +22,13 @@ from inference_endpoint.validation.models import Override, freeze from inference_endpoint.validation.types import EvidenceKey, Severity from inference_endpoint.validation.vocabulary import CheckKind +from tests.unit.validation.helpers import selected_policy pytestmark = pytest.mark.unit -def policy_only(*ids): - policy = load_policy(bundled_policy_path()) - return replace( - policy, checks=tuple(rule for rule in policy.checks if rule.id in ids) - ) - - def test_min_completed_samples_uses_catalog_sample_count(standardized_submission): - policy = policy_only("min-completed-samples") + policy = selected_policy("min-completed-samples") original = execute_submission(standardized_submission, policy) catalog = { **policy.catalogs, @@ -49,7 +43,7 @@ def test_min_completed_samples_uses_catalog_sample_count(standardized_submission def test_duration_catalog_edit_changes_runtime_gate(standardized_submission): - policy = policy_only("region-basis", "point-duration") + policy = selected_policy("region-basis", "point-duration") original = execute_submission(standardized_submission, policy) catalog = { **policy.catalogs, @@ -67,10 +61,8 @@ def test_duration_catalog_edit_changes_runtime_gate(standardized_submission): def test_new_yaml_rule_and_selection_override_execute( - standardized_submission, tmp_path + standardized_submission, policy_dir ): - policy_dir = tmp_path / "2026-10-C1" - shutil.copytree(bundled_policy_path(), policy_dir) point_yaml = policy_dir / "point_checks.yaml" document = yaml.safe_load(point_yaml.read_text()) document["checks"] = { @@ -111,7 +103,7 @@ def test_untrusted_nan_region_basis_blocks_without_crashing( data["max_supported_concurrency"] = float("nan") path.write_text(json.dumps(data)) report = execute_submission( - submission, policy_only("region-computation", "point-duration") + submission, selected_policy("region-computation", "point-duration") ) assert not report.passed @@ -125,7 +117,7 @@ def __call__(self, check, artifacts): def test_model_accuracy_catalog_changes_public_results(standardized_submission): - policy = policy_only("accuracy-gate") + policy = selected_policy("accuracy-gate") profile = { "accuracy": { "kind": "single_turn", @@ -141,9 +133,9 @@ def test_model_accuracy_catalog_changes_public_results(standardized_submission): assert not validate_submission(standardized_submission, policy=stricter).passed -def test_disabled_cohort_override_prevents_execution(standardized_submission, tmp_path): - policy_dir = tmp_path / "2026-10-C1" - shutil.copytree(bundled_policy_path(), policy_dir) +def test_disabled_cohort_override_prevents_execution( + standardized_submission, policy_dir +): catalog_file = policy_dir / "catalog.yaml" catalog = yaml.safe_load(catalog_file.read_text()) catalog["enrollment"]["overrides"] = [ @@ -157,26 +149,21 @@ def test_disabled_cohort_override_prevents_execution(standardized_submission, tm ] catalog_file.write_text(yaml.safe_dump(catalog)) policy = load_policy(policy_dir) - policy = replace( - policy, - checks=tuple( - rule for rule in policy.checks if rule.id == "min-completed-samples" - ), - ) + policy = selected_policy("min-completed-samples", policy=policy) assert validate_submission(standardized_submission, policy=policy).results == [] def test_unregistered_evaluator_blocks(standardized_submission, monkeypatch): monkeypatch.delitem(Evaluator.registry, CheckKind.COMPARISON) report = execute_submission( - standardized_submission, policy_only("metric-consistency-duration") + standardized_submission, selected_policy("metric-consistency-duration") ) assert report.errors assert all(f.key == "blocked" for f in report.errors) def test_effective_override_changes_evaluator_threshold(standardized_submission): - policy = policy_only("point-cap") + policy = selected_policy("point-cap") override = Override( id="smaller-cohort-cap", rule="point-cap", @@ -194,7 +181,7 @@ def test_effective_override_changes_evaluator_threshold(standardized_submission) def test_disabled_region_basis_does_not_publish_dependencies(standardized_submission): - policy = policy_only("region-basis") + policy = selected_policy("region-basis") exemption = Override( id="disabled-basis", rule="region-basis", @@ -222,7 +209,7 @@ def test_region_basis_uses_selected_collection_members( data = yaml.safe_load(path.read_text()) data["offline"] = "dedicated" path.write_text(yaml.safe_dump(data)) - policy = policy_only("region-basis") + policy = selected_policy("region-basis") rule = replace(policy.checks[0], applies_to=Conditions(offline=("none",))) artifacts = load_artifacts(submission, replace(policy, checks=(rule,))) assert next(iter(artifacts.derived.regions.values()))["low_latency"] == (1, 20) @@ -230,7 +217,7 @@ def test_region_basis_uses_selected_collection_members( @pytest.mark.parametrize("rule_id", ["region-placement", "offline-point-present"]) def test_unknown_collection_operation_is_rejected(rule_id): - rule = next(rule for rule in policy_only(rule_id).checks if rule.id == rule_id) + rule = next(rule for rule in selected_policy(rule_id).checks if rule.id == rule_id) with pytest.raises(ValidationError, match="operation"): rule.requirements.with_updates({"operation": "unsupported"}) @@ -244,7 +231,7 @@ def test_offline_uses_effective_numeric_concurrency_floor( data = yaml.safe_load(path.read_text()) data["offline"] = "dedicated" path.write_text(yaml.safe_dump(data)) - policy = policy_only("offline-ordering") + policy = selected_policy("offline-ordering") rule = replace( policy.checks[0], requirements=policy.checks[0].requirements.with_updates( @@ -257,7 +244,7 @@ def test_offline_uses_effective_numeric_concurrency_floor( def test_elected_offline_uses_effective_required_concurrency(standardized_submission): - policy = policy_only("offline-point-present") + policy = selected_policy("offline-point-present") rule = replace( policy.checks[0], requirements=policy.checks[0].requirements.with_updates( @@ -278,7 +265,7 @@ def test_schema_errors_use_effective_warning_severity( data = yaml.safe_load(path.read_text()) data["concurrency"] = "invalid" path.write_text(yaml.safe_dump(data)) - policy = policy_only("point-config-valid") + policy = selected_policy("point-config-valid") rule = replace(policy.checks[0], severity=Severity.WARNING) report = execute_submission(submission, replace(policy, checks=(rule,))) assert report.passed and report.warnings @@ -286,7 +273,7 @@ def test_schema_errors_use_effective_warning_severity( def test_conflicting_region_basis_partitions_block_explicitly(standardized_submission): - policy = policy_only("region-basis", "point-duration") + policy = selected_policy("region-basis", "point-duration") partition = Override( id="different-clamp", rule="region-basis", diff --git a/tests/unit/validation/test_final_review_regressions.py b/tests/unit/validation/test_final_review_regressions.py index 2e5d1b94b..e0b3a8ff3 100644 --- a/tests/unit/validation/test_final_review_regressions.py +++ b/tests/unit/validation/test_final_review_regressions.py @@ -4,14 +4,12 @@ import json import shutil -from dataclasses import replace import pytest import yaml from pydantic import ValidationError from inference_endpoint.validation import ( - bundled_policy_path, load_policy, validate_submission, ) @@ -26,17 +24,11 @@ from inference_endpoint.validation.schemas.requirements_v1 import ( SeedBindingRequirements, ) +from tests.unit.validation.helpers import selected_policy pytestmark = pytest.mark.unit -def selected_policy(*ids): - policy = load_policy(bundled_policy_path()) - return replace( - policy, checks=tuple(rule for rule in policy.checks if rule.id in ids) - ) - - def write_summary(root, summary): point = root / "results/system/gpt-oss-120b/r16" point.mkdir(parents=True) @@ -220,9 +212,8 @@ def test_sample_accounting_includes_failures( @pytest.mark.parametrize("override", [False, True]) -def test_unknown_policy_mode_is_rejected_before_selection(tmp_path, override): - directory = tmp_path / "2026-10-C1" - shutil.copytree(bundled_policy_path(), directory) +def test_unknown_policy_mode_is_rejected_before_selection(policy_dir, override): + directory = policy_dir path = directory / ("catalog.yaml" if override else "curve_checks.yaml") document = yaml.safe_load(path.read_text()) if override: @@ -300,10 +291,8 @@ def test_requirement_sequences_and_aliases_are_immutable(): ], ) def test_sum_operands_support_nesting_defaults_and_missing_values( - tmp_path, operands, issued, passed + tmp_path, policy_dir, operands, issued, passed ): - policy_dir = tmp_path / "2026-10-C1" - shutil.copytree(bundled_policy_path(), policy_dir) path = policy_dir / "point_checks.yaml" document = yaml.safe_load(path.read_text()) document["checks"]["metric-consistency-accounting"]["left"] = { @@ -312,12 +301,7 @@ def test_sum_operands_support_nesting_defaults_and_missing_values( } path.write_text(yaml.safe_dump(document, sort_keys=False)) policy = load_policy(policy_dir) - policy = replace( - policy, - checks=tuple( - rule for rule in policy.checks if rule.id == "metric-consistency-accounting" - ), - ) + policy = selected_policy("metric-consistency-accounting", policy=policy) write_summary(tmp_path, {"n_samples_issued": issued}) report = validate_submission(tmp_path, policy=policy) assert len(report.results) == 1 diff --git a/tests/unit/validation/test_loader.py b/tests/unit/validation/test_loader.py index c711fa814..cae5e4df3 100644 --- a/tests/unit/validation/test_loader.py +++ b/tests/unit/validation/test_loader.py @@ -4,7 +4,6 @@ import hashlib import json -import shutil from pathlib import Path import pytest @@ -28,13 +27,6 @@ def fingerprint(value): ).hexdigest() -@pytest.fixture -def policy_dir(tmp_path): - target = tmp_path / "2026-10-C1" - shutil.copytree(bundled_policy_path(), target) - return target - - def modify(path, transform): data = yaml.safe_load(path.read_text()) transform(data) diff --git a/tests/unit/validation/test_planner.py b/tests/unit/validation/test_planner.py index 7fb77fb49..6d61f66f1 100644 --- a/tests/unit/validation/test_planner.py +++ b/tests/unit/validation/test_planner.py @@ -2,8 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 """Selection, exclusion, dependency and override behavior.""" -import shutil - import pytest import yaml @@ -167,9 +165,8 @@ def test_conditions_and_contexts_reject_contradictory_evidence(): assert condition(point(offline=None)) is Match.UNKNOWN -def test_model_override_changes_only_selected_context(tmp_path): - directory = tmp_path / "2026-10-C1" - shutil.copytree(bundled_policy_path(), directory) +def test_model_override_changes_only_selected_context(policy_dir): + directory = policy_dir path = directory / "catalog.yaml" data = yaml.safe_load(path.read_text()) override = { diff --git a/tests/unit/validation/test_policy_contracts.py b/tests/unit/validation/test_policy_contracts.py index aaea13004..4281a2232 100644 --- a/tests/unit/validation/test_policy_contracts.py +++ b/tests/unit/validation/test_policy_contracts.py @@ -3,7 +3,6 @@ """Malformed submissions and policy values must not disable validation.""" import json -import shutil import pytest import yaml @@ -99,13 +98,6 @@ def test_supplied_decode_head_keeps_approval_checks_selected( assert checks["spec-decode-head-approval-lead-time"].decision is Decision.READY -@pytest.fixture -def policy_dir(tmp_path): - directory = tmp_path / "2026-10-C1" - shutil.copytree(bundled_policy_path(), directory) - return directory - - @pytest.mark.parametrize("value", [[], "false", 0, None]) @pytest.mark.parametrize("override", [False, True]) def test_boolean_requirement_types_are_strict(policy_dir, value, override): diff --git a/tests/unit/validation/test_seed_catalog.py b/tests/unit/validation/test_seed_catalog.py index 89068fb97..66d90e733 100644 --- a/tests/unit/validation/test_seed_catalog.py +++ b/tests/unit/validation/test_seed_catalog.py @@ -36,10 +36,6 @@ def test_mirrors_pr117(self) -> None: seed_set = load_seed_sets()["A"] assert seed_set.seeds == _PR117_SET_A - def test_seeds_are_not_the_v07_constant(self) -> None: - """Published seed fields carry their independent registry values.""" - assert all(value != 42 for value in load_seed_sets()["A"].seeds.values()) - def test_adoption_window_is_derived_from_the_publication_cohort(self) -> None: """§4.6: adoptable for the publication cohort and the three following it.""" assert load_seed_sets()["A"].cohorts == ( @@ -53,74 +49,39 @@ def test_bundled_file_ships_with_the_package(self) -> None: assert bundled_seed_sets_path().is_file() -class TestRegistryShapes: - """Upstream nests under `cohort:`; the older flat form is still read.""" - - def test_upstream_cohort_shape(self, tmp_path: Path) -> None: - path = tmp_path / "seeds.yaml" - path.write_text( - yaml.safe_dump( - { - "cohort": { - "version": 1.0, - "cohort-id": "2027-03-C0", - "seed_sets": [ - { - "id": "Z", - "scheduler_rng_seed": 1, - "sample_index_rng_seed": 2, - "model_seed": 3, - } - ], - } - } - ) - ) - z = parse_seed_catalog(path)["Z"] - assert z.cohorts == ("2027-03-C0", "2027-03-C1", "2027-04-C0", "2027-04-C1") - - def test_flat_shape_still_parses_but_declares_no_window( - self, tmp_path: Path - ) -> None: - """A file predating the cohort wrapper leaves the adoption test unevaluable.""" - path = tmp_path / "seeds.yaml" - path.write_text( - yaml.safe_dump( - { - "seed_sets": [ - { - "id": "Z", - "scheduler_rng_seed": 1, - "sample_index_rng_seed": 2, - "model_seed": 3, - } - ] - } - ) - ) - assert parse_seed_catalog(path)["Z"].cohorts == () - - def test_per_set_cohorts_override_the_derived_window(self, tmp_path: Path) -> None: - path = tmp_path / "seeds.yaml" - path.write_text( - yaml.safe_dump( - { - "cohort": { - "cohort-id": "2026-10-C1", - "seed_sets": [ - { - "id": "Z", - "scheduler_rng_seed": 1, - "sample_index_rng_seed": 2, - "model_seed": 3, - "cohorts": ["2030-01-C0"], - } - ], - } - } - ) - ) - assert parse_seed_catalog(path)["Z"].cohorts == ("2030-01-C0",) +@pytest.mark.parametrize( + "wrapper,cohorts,expected", + [ + pytest.param( + {"version": 1.0, "cohort-id": "2027-03-C0"}, + None, + ("2027-03-C0", "2027-03-C1", "2027-04-C0", "2027-04-C1"), + id="cohort-wrapper", + ), + pytest.param(None, None, (), id="flat-without-window"), + pytest.param( + {"cohort-id": "2026-10-C1"}, + ["2030-01-C0"], + ("2030-01-C0",), + id="explicit-window-overrides-derived", + ), + ], +) +def test_seed_registry_shapes(tmp_path, wrapper, cohorts, expected): + entry = { + "id": "Z", + "scheduler_rng_seed": 1, + "sample_index_rng_seed": 2, + "model_seed": 3, + } + if cohorts is not None: + entry["cohorts"] = cohorts + document = {"seed_sets": [entry]} + if wrapper is not None: + document = {"cohort": {**wrapper, **document}} + path = tmp_path / "seeds.yaml" + path.write_text(yaml.safe_dump(document)) + assert parse_seed_catalog(path)["Z"].cohorts == expected class TestCatalogParsings: diff --git a/tests/unit/validation/test_spec_decode_heads.py b/tests/unit/validation/test_spec_decode_heads.py index 2d5aad99d..165c0118c 100644 --- a/tests/unit/validation/test_spec_decode_heads.py +++ b/tests/unit/validation/test_spec_decode_heads.py @@ -3,8 +3,6 @@ """SpecDecodeHead names and accepted declaration forms preserve approval checks.""" import json -import shutil -from dataclasses import replace import pytest import yaml @@ -20,10 +18,26 @@ SpecDecodeHeadDeclaration, ) from inference_endpoint.validation.schemas.values_v1 import Catalogs +from tests.unit.validation.helpers import selected_policy pytestmark = pytest.mark.unit +def parse_config(**fields): + return PointConfig.model_validate( + {"concurrency": 16, "runtime_settings": {"runtime": {}}, **fields} + ) + + +def write_point(root, model, **fields): + point = root / "results/system" / model / "r16" + point.mkdir(parents=True) + (point / "point.yaml").write_text( + json.dumps({"concurrency": 16, "runtime_settings": {"runtime": {}}, **fields}) + ) + return point + + @pytest.mark.parametrize( "field", ["speculative_decoding", "spec_decode_head", "drafter"] ) @@ -37,23 +51,8 @@ def test_spec_decode_head_declarations_select_and_run_approval( "repository": head["repository"], "revision": head["revision"] if approved else "0" * 40, } - point = tmp_path / "results" / "system" / head["model"] / "r16" - point.mkdir(parents=True) - (point / "point.yaml").write_text( - json.dumps( - { - "concurrency": 16, - "runtime_settings": {"runtime": {}}, - field: declaration, - } - ) - ) - selected = replace( - policy, - checks=tuple( - rule for rule in policy.checks if rule.id == "approved-spec-decode-head" - ), - ) + write_point(tmp_path, head["model"], **{field: declaration}) + selected = selected_policy("approved-spec-decode-head", policy=policy) report = validate_submission(tmp_path, policy=selected) assert len(report.results) == 1 assert report.passed is approved @@ -62,24 +61,14 @@ def test_spec_decode_head_declarations_select_and_run_approval( @pytest.mark.parametrize("canonical", [None, {}, {"repository": "different"}]) def test_conflicting_spec_decode_head_aliases_are_rejected(canonical): with pytest.raises(ValidationError, match="input alias must agree"): - PointConfig.model_validate( - { - "concurrency": 16, - "runtime_settings": {"runtime": {}}, - "spec_decode_head": canonical, - "drafter": {"repository": "head", "revision": "a" * 40}, - } + parse_config( + spec_decode_head=canonical, + drafter={"repository": "head", "revision": "a" * 40}, ) def test_spec_decode_head_input_alias_serializes_with_canonical_name(): - parsed = PointConfig.model_validate( - { - "concurrency": 16, - "runtime_settings": {"runtime": {}}, - "drafter": {"repository": "head", "revision": "a" * 40}, - } - ) + parsed = parse_config(drafter={"repository": "head", "revision": "a" * 40}) payload = parsed.model_dump(mode="json", exclude_unset=True) assert payload["spec_decode_head"] == {"repository": "head", "revision": "a" * 40} assert "drafter" not in payload @@ -95,31 +84,16 @@ def test_published_weight_checksums_and_approval_lead_time( policy = load_policy(bundled_policy_path()) head = policy.catalogs["approved_spec_decode_heads"][head_index] assert head["approved_cohort"] == "2026-09-C1" - point = tmp_path / "results/system" / head["model"] / "r16" - point.mkdir(parents=True) - (point / "point.yaml").write_text( - json.dumps( - { - "concurrency": 16, - "runtime_settings": {"runtime": {}}, - "target_cohort": target, - "speculative_decoding": { - "weight_checksum": "git-sha1:" + head["revision"] - }, - } - ) + write_point( + tmp_path, + head["model"], + target_cohort=target, + speculative_decoding={"weight_checksum": "git-sha1:" + head["revision"]}, ) - selected = replace( - policy, - checks=tuple( - rule - for rule in policy.checks - if rule.id - in { - "approved-spec-decode-head", - "spec-decode-head-approval-lead-time", - } - ), + selected = selected_policy( + "approved-spec-decode-head", + "spec-decode-head-approval-lead-time", + policy=policy, ) report = validate_submission(tmp_path, policy=selected) assert len(report.results) == 2 @@ -142,13 +116,7 @@ def test_published_weight_checksums_and_approval_lead_time( ) def test_malformed_or_conflicting_weight_identity_is_rejected(identity): with pytest.raises(ValidationError): - PointConfig.model_validate( - { - "concurrency": 16, - "runtime_settings": {"runtime": {}}, - "spec_decode_head": identity, - } - ) + parse_config(spec_decode_head=identity) @pytest.mark.parametrize( @@ -174,23 +142,8 @@ def test_weight_identity_must_match_model_and_supplied_repository( if "target_checksum" not in identity: checksum_revision = identity.get("revision", head["revision"]) identity = {"weight_checksum": "git-sha1:" + checksum_revision, **identity} - point = tmp_path / "results/system" / (model or head["model"]) / "r16" - point.mkdir(parents=True) - (point / "point.yaml").write_text( - json.dumps( - { - "concurrency": 16, - "runtime_settings": {"runtime": {}}, - "spec_decode_head": identity, - } - ) - ) - selected = replace( - policy, - checks=tuple( - rule for rule in policy.checks if rule.id == "approved-spec-decode-head" - ), - ) + write_point(tmp_path, model or head["model"], spec_decode_head=identity) + selected = selected_policy("approved-spec-decode-head", policy=policy) report = validate_submission(tmp_path, policy=selected) assert len(report.errors) == 1 assert not report.passed @@ -202,13 +155,7 @@ def test_weight_checksum_round_trip_preserves_string_contract(): "repository": "head", "revision": "a" * 40, } - point = PointConfig.model_validate( - { - "concurrency": 16, - "runtime_settings": {"runtime": {}}, - "spec_decode_head": identity, - } - ) + point = parse_config(spec_decode_head=identity) assert point.spec_decode_head.weight_checksum.revision == "a" * 40 assert ( point.model_dump(mode="json", exclude_unset=True)["spec_decode_head"] @@ -229,10 +176,10 @@ def test_weight_checksum_round_trip_preserves_string_contract(): ("cohort", False), ], ) -def test_configuration_identity_approval_and_age(tmp_path, checksum, change, passed): +def test_configuration_identity_approval_and_age( + tmp_path, policy_dir, checksum, change, passed +): # Synthetic policy entry exercises support without adding an official approval. - policy_dir = tmp_path / "2026-10-C1" - shutil.copytree(bundled_policy_path(), policy_dir) catalog_path = policy_dir / "catalog.yaml" document = yaml.safe_load(catalog_path.read_text()) model = document["catalogs"]["approved_spec_decode_heads"][0]["model"] @@ -259,29 +206,16 @@ def test_configuration_identity_approval_and_age(tmp_path, checksum, change, pas declaration["configuration"] = {**configuration, "layers": [1.0, 2]} if change == "model": model = "deepseek-v4_1-flash" - point = tmp_path / "submission/results/system" / model / "r16" - point.mkdir(parents=True) - (point / "point.yaml").write_text( - json.dumps( - { - "concurrency": 16, - "runtime_settings": {"runtime": {}}, - "target_cohort": "2026-10-C0" if change == "cohort" else "2026-10-C1", - "speculative_decoding": declaration, - } - ) + point = write_point( + tmp_path / "submission", + model, + target_cohort="2026-10-C0" if change == "cohort" else "2026-10-C1", + speculative_decoding=declaration, ) - selected = replace( - policy, - checks=tuple( - rule - for rule in policy.checks - if rule.id - in { - "approved-spec-decode-head", - "spec-decode-head-approval-lead-time", - } - ), + selected = selected_policy( + "approved-spec-decode-head", + "spec-decode-head-approval-lead-time", + policy=policy, ) report = validate_submission(tmp_path / "submission", policy=selected) assert report.passed is passed @@ -313,13 +247,7 @@ def test_configuration_identity_approval_and_age(tmp_path, checksum, change, pas ) def test_incomplete_or_mixed_configuration_identity_is_rejected(identity): with pytest.raises(ValidationError): - PointConfig.model_validate( - { - "concurrency": 16, - "runtime_settings": {"runtime": {}}, - "speculative_decoding": identity, - } - ) + parse_config(speculative_decoding=identity) def test_bundled_catalog_has_no_configuration_based_approvals(): diff --git a/tests/unit/validation/test_special_evaluators.py b/tests/unit/validation/test_special_evaluators.py index 2d89cbbbe..0a41e4c63 100644 --- a/tests/unit/validation/test_special_evaluators.py +++ b/tests/unit/validation/test_special_evaluators.py @@ -32,21 +32,11 @@ from inference_endpoint.validation.power.calculation import PowerCalculator from inference_endpoint.validation.power.models import SystemPower from inference_endpoint.validation.types import Decision, EvidenceKey, Scope +from tests.unit.validation.helpers import update_evidence pytestmark = pytest.mark.unit -def update_evidence(point, field, **updates): - artifact = getattr(point.evidence, field) - data = artifact.value.model_dump( - mode="json", exclude_unset=True, exclude_computed_fields=True - ) - data.update(updates) - parsed = ParsedArtifact.from_json(data, type(artifact.value), point.path, field) - assert not parsed.errors - point.evidence = replace(point.evidence, **{field: parsed}) - - def update_accuracy(point, data): parsed = ParsedArtifact.from_json( data, AccuracyResult, point.path, "accuracy-valid" From 67536ba8b897df3f9af49fb06cab162b73f980c2 Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 14:31:45 -0700 Subject: [PATCH 17/18] refactor(validation): remove unused helpers and consolidate registration --- .../validation/checks/__init__.py | 30 +++++++++++++-- src/inference_endpoint/validation/engine.py | 7 ---- .../validation/evidence/loaders.py | 33 ----------------- .../validation/evidence/point_summary.py | 37 ------------------- .../validation/evidence/steady_state.py | 7 ---- src/inference_endpoint/validation/planner.py | 7 +++- tests/unit/validation/test_engine.py | 14 +++++-- tests/unit/validation/test_evidence.py | 21 +++++------ tests/unit/validation/test_loader.py | 14 ++++--- tests/unit/validation/test_runtime_safety.py | 11 +++--- .../validation/test_special_evaluators.py | 1 - 11 files changed, 66 insertions(+), 116 deletions(-) diff --git a/src/inference_endpoint/validation/checks/__init__.py b/src/inference_endpoint/validation/checks/__init__.py index fc6f3be38..021113e17 100644 --- a/src/inference_endpoint/validation/checks/__init__.py +++ b/src/inference_endpoint/validation/checks/__init__.py @@ -1,7 +1,31 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Register built-in callable evaluators by their policy kind.""" +"""Importing this package registers all built-in evaluators by policy kind.""" -from . import accuracy, power, seeds, spec_decode_heads, steady_state, warmup +from .. import bindings +from . import ( + accuracy, + collections, + datasets, + metrics, + power, + seeds, + spec_decode_heads, + steady_state, + structure, + warmup, +) -__all__ = ["accuracy", "spec_decode_heads", "power", "seeds", "steady_state", "warmup"] +__all__ = [ + "accuracy", + "bindings", + "collections", + "datasets", + "metrics", + "power", + "seeds", + "spec_decode_heads", + "steady_state", + "structure", + "warmup", +] diff --git a/src/inference_endpoint/validation/engine.py b/src/inference_endpoint/validation/engine.py index 75760f6b4..a4d64ede1 100644 --- a/src/inference_endpoint/validation/engine.py +++ b/src/inference_endpoint/validation/engine.py @@ -7,14 +7,7 @@ from os import PathLike from pathlib import Path -from . import bindings # noqa: F401 - register evaluator classes from .artifacts import load_artifacts -from .checks import ( # noqa: F401 - register evaluator classes - collections, - datasets, - metrics, - structure, -) from .checks.power import prepare from .evaluator_base import Evaluator from .models import Policy diff --git a/src/inference_endpoint/validation/evidence/loaders.py b/src/inference_endpoint/validation/evidence/loaders.py index 8655cc4bd..b6e3f5912 100644 --- a/src/inference_endpoint/validation/evidence/loaders.py +++ b/src/inference_endpoint/validation/evidence/loaders.py @@ -13,10 +13,6 @@ from pydantic import BaseModel, ValidationError from ..results import CheckResult -from .accuracy import AccuracyResult -from .point_config import PointConfig -from .point_summary import PointSummary -from .system import SystemDescription MAX_ARTIFACT_DEPTH = 100 @@ -166,32 +162,3 @@ def read_artifact( # noqa: UP047 - supported by the repository type checker return ParsedArtifact(None, frozenset(), (), False) data = {"accuracy_scores": data} return ParsedArtifact.from_json(data, model, path, rule) - - -def load_point_config(path: Path) -> tuple[PointConfig | None, list[CheckResult]]: - parsed = read_artifact(path, PointConfig, "point-config-valid") - return parsed.value, list(parsed.errors) - - -def load_result_summary(path: Path) -> tuple[PointSummary | None, list[CheckResult]]: - parsed = read_artifact(path, PointSummary, "result-file-valid") - return parsed.value, list(parsed.errors) - - -def load_system_description( - path: Path, -) -> tuple[SystemDescription | None, list[CheckResult]]: - parsed = read_artifact(path, SystemDescription, "system-description-valid") - return parsed.value, list(parsed.errors) - - -def load_accuracy_result(path: Path) -> tuple[AccuracyResult | None, list[CheckResult]]: - parsed = read_artifact(path, AccuracyResult, "accuracy-valid") - return parsed.value, list(parsed.errors) - - -def load_accuracy_scores( - path: Path, -) -> tuple[AccuracyResult | None, list[CheckResult], bool]: - parsed = read_artifact(path, AccuracyResult, "accuracy-valid", accuracy_scores=True) - return parsed.value, list(parsed.errors), parsed.present diff --git a/src/inference_endpoint/validation/evidence/point_summary.py b/src/inference_endpoint/validation/evidence/point_summary.py index 735e30ba4..d4397f561 100644 --- a/src/inference_endpoint/validation/evidence/point_summary.py +++ b/src/inference_endpoint/validation/evidence/point_summary.py @@ -77,18 +77,6 @@ class PointSummary(BaseModel): output_tokens_per_turn_total: FiniteFloat | None = None e2e_turn_time_seconds_total: FiniteFloat | None = None - @computed_field # type: ignore[misc] - @property - def duration_ms(self) -> float: - """Measurement duration in milliseconds.""" - return self.duration_ns / 1000000 - - @computed_field # type: ignore[misc] - @property - def sample_count(self) -> int: - """Alias for ``n_samples_completed``.""" - return self.n_samples_completed - @computed_field # type: ignore[misc] @property def total_output_tokens(self) -> int: @@ -108,31 +96,6 @@ def system_tps(self) -> float: elapsed_s = self.elapsed_duration_seconds return self.total_output_tokens / elapsed_s if elapsed_s > 0 else 0.0 - @computed_field # type: ignore[misc] - @property - def ttft_p50_ms(self) -> float: - """Median time to first token in milliseconds.""" - return self.ttft.percentiles.get("50", 0.0) / 1000000 - - @computed_field # type: ignore[misc] - @property - def ttft_p90_ms(self) -> float: - """90th-percentile time to first token in milliseconds (§4.1).""" - return self.ttft.percentiles.get("90", 0.0) / 1000000 - - @computed_field # type: ignore[misc] - @property - def ttft_p95_ms(self) -> float: - """95th-percentile time to first token in milliseconds.""" - return self.ttft.percentiles.get("95", 0.0) / 1000000 - - @computed_field # type: ignore[misc] - @property - def tpot_p90_ms(self) -> float | None: - """90th-percentile time per output token in milliseconds (§9.1).""" - raw = self.tpot.percentiles.get("90") - return None if raw is None else raw / 1000000 - @computed_field # type: ignore[misc] @property def e2e_avg_interactivity(self) -> float | None: diff --git a/src/inference_endpoint/validation/evidence/steady_state.py b/src/inference_endpoint/validation/evidence/steady_state.py index 6ddb35fd1..5e569d4ae 100644 --- a/src/inference_endpoint/validation/evidence/steady_state.py +++ b/src/inference_endpoint/validation/evidence/steady_state.py @@ -69,13 +69,6 @@ class SteadyStateWindow(BaseModel): n_samples: int | None = None duration_s: float | None = None - @property - def super_passes(self) -> int | None: - """How many super-passes the window spans, or None when unstated.""" - if self.super_pass_start is None or self.super_pass_end is None: - return None - return self.super_pass_end - self.super_pass_start + 1 - class SteadyState(BaseModel): """A point's §4.4 reporting block.""" diff --git a/src/inference_endpoint/validation/planner.py b/src/inference_endpoint/validation/planner.py index 52c4b5fc7..3fe7b37bb 100644 --- a/src/inference_endpoint/validation/planner.py +++ b/src/inference_endpoint/validation/planner.py @@ -75,9 +75,12 @@ def _member_subject(subject: Context, member: Context) -> Context: ) -def _select_collection(rule: Rule, subject: Context): +def _select_collection( + rule: Rule, subject: Context +) -> tuple[Decision, str, tuple[str, ...] | None]: if subject.members is None or not (rule.applies_to or rule.unless): - return (*_select(rule, subject), None) + decision, reason = _select(rule, subject) + return decision, reason, None selected = [] unknown = False for member in subject.members: diff --git a/tests/unit/validation/test_engine.py b/tests/unit/validation/test_engine.py index df61d7bb3..bf15ccb66 100644 --- a/tests/unit/validation/test_engine.py +++ b/tests/unit/validation/test_engine.py @@ -108,12 +108,18 @@ def test_untrusted_nan_region_basis_blocks_without_crashing( assert not report.passed +def test_all_builtin_kinds_are_registered(): + assert set(Evaluator.registry) == set(CheckKind) + + def test_registry_rejects_duplicate_kinds(): with pytest.raises(ValueError, match="Duplicate evaluator"): - - class Duplicate(Evaluator, kind=CheckKind.COMPARISON): - def __call__(self, check, artifacts): - return [] + type( + "Duplicate", + (Evaluator,), + {"__call__": lambda self, check, artifacts: []}, + kind=CheckKind.COMPARISON, + ) def test_model_accuracy_catalog_changes_public_results(standardized_submission): diff --git a/tests/unit/validation/test_evidence.py b/tests/unit/validation/test_evidence.py index fe89379ee..6f7acdef3 100644 --- a/tests/unit/validation/test_evidence.py +++ b/tests/unit/validation/test_evidence.py @@ -18,10 +18,9 @@ parse_seed_catalog, ) from inference_endpoint.validation.evidence.accuracy import AccuracyResult -from inference_endpoint.validation.evidence.loaders import ( - load_point_config, - load_result_summary, -) +from inference_endpoint.validation.evidence.loaders import read_artifact +from inference_endpoint.validation.evidence.point_config import PointConfig +from inference_endpoint.validation.evidence.point_summary import PointSummary from inference_endpoint.validation.power.models import SourcedValue pytestmark = pytest.mark.unit @@ -30,10 +29,10 @@ def test_point_parsing_does_not_enforce_submission_policy(tmp_path): path = tmp_path / "point.yaml" path.write_text("concurrency: 1\nruntime_settings:\n runtime: {}\n") - model, findings = load_point_config(path) - assert model is not None - assert not findings - assert not hasattr(model, "_check_results") + parsed = read_artifact(path, PointConfig, "point-config-valid") + assert parsed.value is not None + assert not parsed.errors + assert not hasattr(parsed.value, "_check_results") def test_native_accuracy_aliases_and_duplicate_datasets(): @@ -65,9 +64,9 @@ def test_empty_accuracy_is_rejected_by_selected_check(tmp_path): def test_unreadable_encoding_becomes_structured_schema_error(tmp_path): path = tmp_path / "result_summary.json" path.write_bytes(b"\xff") - model, findings = load_result_summary(path) - assert model is None - assert findings[0].key == "artifact-unreadable" + parsed = read_artifact(path, PointSummary, "result-file-valid") + assert parsed.value is None + assert parsed.errors[0].key == "artifact-unreadable" @pytest.mark.parametrize("value", [float("inf"), float("nan")]) diff --git a/tests/unit/validation/test_loader.py b/tests/unit/validation/test_loader.py index cae5e4df3..32b63652d 100644 --- a/tests/unit/validation/test_loader.py +++ b/tests/unit/validation/test_loader.py @@ -164,12 +164,14 @@ def test_parser_dispatch_has_no_latest_revision_fallback(): with pytest.raises(ValueError, match="Unsupported"): parser_for(Release(version=version, revision=revision)) with pytest.raises(ValueError, match="Overlapping"): - - class Conflicting( - BundleParser, version="2026-10-C1", first_revision=1, last_revision=2 - ): - def __call__(self, documents, digests): - raise AssertionError("Must not execute") + type( + "Conflicting", + (BundleParser,), + {}, + version="2026-10-C1", + first_revision=1, + last_revision=2, + ) @pytest.mark.parametrize( diff --git a/tests/unit/validation/test_runtime_safety.py b/tests/unit/validation/test_runtime_safety.py index 8481ff818..be30a3b5d 100644 --- a/tests/unit/validation/test_runtime_safety.py +++ b/tests/unit/validation/test_runtime_safety.py @@ -6,7 +6,8 @@ import pytest -from inference_endpoint.validation.evidence.loaders import load_result_summary +from inference_endpoint.validation.evidence.loaders import read_artifact +from inference_endpoint.validation.evidence.point_summary import PointSummary from inference_endpoint.validation.types import Severity pytestmark = pytest.mark.unit @@ -23,7 +24,7 @@ def test_invalid_stored_metrics_are_structured_load_errors(tmp_path, field, valu path.write_text( json.dumps({"n_samples_completed": 1, "duration_ns": 1e9, field: value}) ) - model, results = load_result_summary(path) - assert model is None - assert results - assert all(result.severity == Severity.ERROR for result in results) + parsed = read_artifact(path, PointSummary, "result-file-valid") + assert parsed.value is None + assert parsed.errors + assert all(result.severity == Severity.ERROR for result in parsed.errors) diff --git a/tests/unit/validation/test_special_evaluators.py b/tests/unit/validation/test_special_evaluators.py index 0a41e4c63..6d304d1d5 100644 --- a/tests/unit/validation/test_special_evaluators.py +++ b/tests/unit/validation/test_special_evaluators.py @@ -13,7 +13,6 @@ Conditions, Context, bundled_policy_path, - checks, # noqa: F401 - registers built-in evaluators load_policy, ) from inference_endpoint.validation.artifacts import ( From 615ea768063f8a1a8e42e3c7cf7f3231b7dc5ead Mon Sep 17 00:00:00 2001 From: Alice Cheng Date: Thu, 8 Oct 2026 14:43:27 -0700 Subject: [PATCH 18/18] style(validation): use pass for empty overload bodies --- src/inference_endpoint/validation/models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/inference_endpoint/validation/models.py b/src/inference_endpoint/validation/models.py index 5eefec338..8882c6294 100644 --- a/src/inference_endpoint/validation/models.py +++ b/src/inference_endpoint/validation/models.py @@ -18,11 +18,13 @@ @overload def freeze( # noqa: UP047 - supported by the repository type checker value: Mapping[_Key, Any], -) -> Mapping[_Key, Any]: ... +) -> Mapping[_Key, Any]: + pass @overload -def freeze(value: object) -> Any: ... +def freeze(value: object) -> Any: + pass def freeze(value: object) -> Any: