diff --git a/tests/integration/defs/perf/agentx_client.py b/tests/integration/defs/perf/agentx_client.py index 04638663d5c7..0cabeed19835 100644 --- a/tests/integration/defs/perf/agentx_client.py +++ b/tests/integration/defs/perf/agentx_client.py @@ -105,6 +105,10 @@ # ("Feature type 'Json' not found"). 4.x+ is required. MIN_DATASETS_MAJOR = 4 +# The aiperf scenario that defines the AgentX workload. Only the agentx-harness +# build registers it. +AGENTX_SCENARIO = "inferencex-agentx-mvp" + # PyExecutor's HangDetector is hard-coded at 300 s with no env override, so a # multi-minute pip install with zero requests in flight idles the GEN worker # into MPI_Abort -- which then presents as a *client* connection failure. A @@ -174,19 +178,43 @@ def _env_float(name: str, default: float) -> float: def aiperf_is_usable() -> bool: """Return whether the ambient interpreter already has a working agentx stack. - Checks importability *and* the datasets major version, since a container - with aiperf but datasets 3.x fails only later, when loading the corpus. + Checks three things, each of which otherwise fails only once the benchmark + starts: + + * the modules import; + * the ``aiperf`` found is the agentx-harness fork. The upstream PyPI + ``aiperf`` (pinned in requirements-dev.txt) has the same import name but + no ``inferencex-agentx-mvp`` scenario, and rejects ``--scenario``; + * datasets is 4.x or newer, since datasets 3.x cannot load the corpus. + + The reason for a failed check is logged. """ probe = ( - "import aiperf, cyclopts, datasets, transformers, pydantic, msgspec, orjson; " - f"import sys; sys.exit(0 if int(datasets.__version__.split('.')[0]) >= {MIN_DATASETS_MAJOR} else 1)" + "import sys\n" + "try:\n" + " import aiperf, cyclopts, datasets, transformers, pydantic, msgspec, orjson\n" + "except ImportError as e:\n" + " sys.exit(f'missing module: {e}')\n" + "try:\n" + " from aiperf.common.scenario.registry import get_scenario\n" + f" get_scenario({AGENTX_SCENARIO!r})\n" + "except Exception as e:\n" + f" sys.exit(f'aiperf is not the agentx-harness build (no {AGENTX_SCENARIO} " + "scenario): {type(e).__name__}: {e}')\n" + f"if int(datasets.__version__.split('.')[0]) < {MIN_DATASETS_MAJOR}:\n" + f" sys.exit(f'datasets {{datasets.__version__}} < {MIN_DATASETS_MAJOR}')\n" ) - return ( - subprocess.call( - [sys.executable, "-c", probe], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL - ) - == 0 + result = subprocess.run( + [sys.executable, "-c", probe], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + timeout=300, ) + if result.returncode != 0: + lines = result.stderr.strip().splitlines() + _log(f"aiperf stack not usable: {lines[-1] if lines else f'exit {result.returncode}'}") + return result.returncode == 0 def post_one_token(url: str, model: str, timeout: float) -> "str | None": @@ -307,7 +335,7 @@ def build_aiperf_cmd(args: argparse.Namespace, url: str, artifact_dir: str) -> " "aiperf", "profile", "--scenario", - "inferencex-agentx-mvp", + AGENTX_SCENARIO, "-m", args.model, "--tokenizer-trust-remote-code", diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index f8bdd7252706..c82c4a005432 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -216,7 +216,6 @@ full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::Tes full:sm100/unittest/bindings SKIP (Disable for Blackwell) llmapi/test_llm_examples.py::test_llmapi_speculative_decoding_eagle3 SKIP (https://nvbugs/6075431) perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6912139) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-dspark_agentx_con1456_ctx3_dep8_gen1_dep16_eplb0_dspark5_ccb-NIXL] SKIP (https://nvbugs/6939002) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] SKIP (https://nvbugs/6581075) perf/test_visual_gen_perf_sanity.py::test_visual_gen_e2e[vg_upload-flux2_blackwell-flux2_fp8_cfg1_ulysses4_teacache_on] SKIP (https://nvbugs/6759009) test_doc.py::test_relative_path_validity SKIP (https://nvbugs/6784181) diff --git a/tests/unittest/scripts/test_perf_agentx_aiperf_probe.py b/tests/unittest/scripts/test_perf_agentx_aiperf_probe.py new file mode 100644 index 000000000000..beef4abfaba8 --- /dev/null +++ b/tests/unittest/scripts/test_perf_agentx_aiperf_probe.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""``agentx_client.aiperf_is_usable`` tells the agentx-harness aiperf from PyPI's. + +Each case builds a fake site directory holding stub modules and runs the real +probe in a subprocess against it, so no aiperf has to be installed. +""" + +import importlib.util +import sys +import textwrap +from pathlib import Path +from types import ModuleType + +import pytest + +pytestmark = pytest.mark.cpu_only + +REPO_ROOT = Path(__file__).resolve().parents[3] +AGENTX_CLIENT_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "perf" / "agentx_client.py" + +# Every module the probe imports besides aiperf itself. +_STUB_MODULES = ("cyclopts", "transformers", "pydantic", "msgspec", "orjson") + +_FORK_REGISTRY = textwrap.dedent( + """ + SCENARIOS = {"inferencex-agentx-mvp": object()} + + def get_scenario(name): + if name not in SCENARIOS: + raise KeyError(name) + return SCENARIOS[name] + """ +) + + +@pytest.fixture +def client() -> ModuleType: + spec = importlib.util.spec_from_file_location("perf_agentx_client", AGENTX_CLIENT_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _make_site( + root: Path, *, fork: bool, registry: str = _FORK_REGISTRY, datasets_version: str = "4.1.0" +) -> Path: + for name in _STUB_MODULES: + (root / f"{name}.py").write_text("") + (root / "datasets.py").write_text(f"__version__ = {datasets_version!r}\n") + pkg = root / "aiperf" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + if fork: + scenario = pkg / "common" / "scenario" + scenario.mkdir(parents=True) + (pkg / "common" / "__init__.py").write_text("") + (scenario / "__init__.py").write_text("") + (scenario / "registry.py").write_text(registry) + return root + + +@pytest.fixture +def run_probe(client, monkeypatch, capfd): + def _run(site: Path) -> "tuple[bool, str]": + monkeypatch.setenv("PYTHONPATH", str(site)) + monkeypatch.setenv("PYTHONNOUSERSITE", "1") + # -S keeps the real site-packages (and any real aiperf) out of the probe. + monkeypatch.setattr(client.sys, "executable", sys.executable) + real_run = client.subprocess.run + + def _isolated(cmd, *args, **kwargs): + return real_run([cmd[0], "-S", *cmd[1:]], *args, **kwargs) + + monkeypatch.setattr(client.subprocess, "run", _isolated) + usable = client.aiperf_is_usable() + return usable, capfd.readouterr().out + + return _run + + +def test_fork_build_is_usable(tmp_path, run_probe): + usable, out = run_probe(_make_site(tmp_path, fork=True)) + assert usable, out + assert out == "" + + +def test_pypi_aiperf_is_not_usable(tmp_path, run_probe): + """The PyPI aiperf imports fine but has no scenario registry.""" + usable, out = run_probe(_make_site(tmp_path, fork=False)) + assert not usable + assert "not the agentx-harness build" in out + assert "inferencex-agentx-mvp" in out + + +def test_fork_without_agentx_scenario_is_not_usable(tmp_path, run_probe): + registry = _FORK_REGISTRY.replace('"inferencex-agentx-mvp"', '"other-scenario"') + usable, out = run_probe(_make_site(tmp_path, fork=True, registry=registry)) + assert not usable + assert "not the agentx-harness build" in out + + +def test_old_datasets_is_not_usable(tmp_path, run_probe): + usable, out = run_probe(_make_site(tmp_path, fork=True, datasets_version="3.1.0")) + assert not usable + assert "datasets 3.1.0 < 4" in out + + +def test_missing_module_is_not_usable(tmp_path, run_probe): + site = _make_site(tmp_path, fork=True) + (site / "orjson.py").unlink() + usable, out = run_probe(site) + assert not usable + assert "missing module" in out + + +def test_command_uses_the_probed_scenario(client): + args = client.parse_args( + ["--model", "m", "--concurrency", "8", "--dataset", "d", "--host", "h", "--port", "1"] + ) + cmd = client.build_aiperf_cmd(args, "http://h:1", "/tmp/out") + assert cmd[cmd.index("--scenario") + 1] == client.AGENTX_SCENARIO