Skip to content

Commit b833b7c

Browse files
BowenFuchenfeiz0326claude
authored
[https://nvbugs/6958731][chore] Report benchmark readiness request rejections (#20749)
Signed-off-by: Bowen Fu <5812640+BowenFu@users.noreply.github.com> Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Co-authored-by: Bowen Fu <5812640+BowenFu@users.noreply.github.com> Co-authored-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com>
1 parent f806023 commit b833b7c

3 files changed

Lines changed: 137 additions & 12 deletions

File tree

‎tests/integration/defs/perf/agentx_client.py‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -218,8 +218,14 @@ def aiperf_is_usable() -> bool:
218218
return result.returncode == 0
219219

220220

221-
def post_one_token(url: str, model: str, timeout: float) -> "str | None":
222-
"""POST a 1-token non-streaming completion. Returns the body, or None on failure."""
221+
def post_one_token(
222+
url: str, model: str, timeout: float, *, fail_on_client_error: bool = False
223+
) -> "str | None":
224+
"""POST one token, optionally raising with the reason for a rejected request.
225+
226+
Transport failures, server errors, and retryable HTTP 408/429 responses
227+
return None. Heartbeats also ignore request rejections by default.
228+
"""
223229
payload = json.dumps(
224230
{
225231
"model": model,
@@ -237,6 +243,15 @@ def post_one_token(url: str, model: str, timeout: float) -> "str | None":
237243
try:
238244
with urllib.request.urlopen(req, timeout=timeout) as resp:
239245
return resp.read().decode(errors="replace")
246+
except urllib.error.HTTPError as error:
247+
if fail_on_client_error and 400 <= error.code < 500 and error.code not in (408, 429):
248+
with error:
249+
body = error.read(2048).decode(errors="replace")
250+
raise RuntimeError(
251+
f"{req.full_url} rejected the 1-token readiness request: HTTP {error.code}: {body}"
252+
) from error
253+
error.close()
254+
return None
240255
except (urllib.error.URLError, OSError, TimeoutError):
241256
return None
242257

@@ -300,27 +315,25 @@ def ensure_aiperf(url: str, model: str) -> None:
300315
def verify_endpoint(url: str, model: str) -> None:
301316
"""Prove the endpoint serves one real token before spending GPU-hours.
302317
303-
The harness has already waited for the disagg server, so this is a short
304-
confirmation rather than the reference script's 5400 s poll. It exists
305-
because a broken ctx->gen KV path answers /health perfectly well and only
306-
fails once real traffic arrives -- an hour later, as a client-side error.
318+
A healthy process may still reject inference requests or fail to generate.
319+
This check applies to both aggregated and disaggregated servers. Permanent
320+
request errors fail immediately with the response body; transient failures
321+
retain the readiness retry budget.
307322
"""
308323
timeout = _env_int("AGENTX_READY_TIMEOUT", 900)
309324
interval = _env_int("AGENTX_READY_INTERVAL", 15)
310325
deadline = time.monotonic() + timeout
311326
last = None
312327
while time.monotonic() < deadline:
313-
last = post_one_token(url, model, timeout=300)
328+
last = post_one_token(url, model, timeout=300, fail_on_client_error=True)
314329
if last and '"choices"' in last:
315-
_log("1-token completion OK -- the disagg path is live")
330+
_log("1-token completion OK -- the inference endpoint is ready")
316331
return
317332
_log(f"not serving yet (response head: {str(last)[:200]})")
318333
time.sleep(interval)
319334
raise RuntimeError(
320-
f"{url} could not complete ONE token within {timeout}s. This is a "
321-
f"SERVER-side failure: inspect the ctx/gen worker logs and the disagg "
322-
f"server log, not the client. Suspect the ctx->gen KV transfer "
323-
f"(on GB300/NVL72 check UCX_TLS contains no 'rc'). "
335+
f"{url} could not complete ONE token within {timeout}s. "
336+
f"Inspect the inference server logs and readiness request failures. "
324337
f"Last response: {str(last)[:500]}"
325338
)
326339

‎tests/integration/test_lists/test-db/l0_cpu.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ l0_cpu:
3939
# catches a pruned reference before the post-merge canary does.
4040
- model_express/test_model_express_accuracy.py::test_mx_accuracy_cases_have_references
4141
- unittest/test_s3_output.py
42+
- unittest/scripts/test_perf_client_readiness.py
4243
- unittest/_torch/visual_gen/multi_gpu/test_worker_pool.py
4344
# Covers tests/unittest/_torch/attention/. Two sub-trees moved in here from elsewhere
4445
# under tests/unittest/_torch/ and have no entry of their own on any list, so this entry
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
"""Exercise benchmark client readiness against real HTTP rejection and retry responses."""
16+
17+
import importlib.util
18+
import json
19+
import threading
20+
import urllib.error
21+
from http.server import BaseHTTPRequestHandler, HTTPServer
22+
from pathlib import Path
23+
from types import ModuleType
24+
25+
import pytest
26+
27+
pytestmark = pytest.mark.cpu_only
28+
29+
CLIENT_PATH = Path(__file__).resolve().parents[3] / "tests/integration/defs/perf/agentx_client.py"
30+
31+
32+
@pytest.fixture
33+
def client(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
34+
spec = importlib.util.spec_from_file_location("perf_benchmark_client", CLIENT_PATH)
35+
assert spec is not None and spec.loader is not None
36+
module = importlib.util.module_from_spec(spec)
37+
spec.loader.exec_module(module)
38+
monkeypatch.setenv("AGENTX_READY_TIMEOUT", "60")
39+
monkeypatch.setattr(module.time, "sleep", lambda _: None)
40+
return module
41+
42+
43+
@pytest.fixture
44+
def endpoint():
45+
responses = []
46+
requests = []
47+
48+
class Handler(BaseHTTPRequestHandler):
49+
def do_POST(self) -> None:
50+
requests.append(
51+
(self.path, json.loads(self.rfile.read(int(self.headers["Content-Length"]))))
52+
)
53+
status, body = responses.pop(0) if responses else (503, "not ready")
54+
self.send_response(status)
55+
self.end_headers()
56+
self.wfile.write(body.encode())
57+
58+
def log_message(self, *args: object) -> None:
59+
pass
60+
61+
server = HTTPServer(("127.0.0.1", 0), Handler)
62+
thread = threading.Thread(target=server.serve_forever, daemon=True)
63+
thread.start()
64+
try:
65+
yield f"http://127.0.0.1:{server.server_port}", responses, requests
66+
finally:
67+
server.shutdown()
68+
server.server_close()
69+
thread.join(timeout=5)
70+
71+
72+
@pytest.mark.parametrize("status", [400, 401, 403, 404, 422])
73+
def test_rejection_preserves_server_reason(client: ModuleType, endpoint, status: int) -> None:
74+
url, responses, requests = endpoint
75+
responses.append((status, '{"error":{"message":"unsupported chat template"}}'))
76+
with pytest.raises(RuntimeError, match=f"HTTP {status}.*unsupported chat template"):
77+
client.verify_endpoint(url, "model")
78+
assert len(requests) == 1
79+
assert requests[0][0] == "/v1/chat/completions"
80+
assert requests[0][1]["model"] == "model"
81+
82+
83+
@pytest.mark.parametrize("status", [408, 429, 500, 503])
84+
def test_transient_http_failure_still_retries(client: ModuleType, endpoint, status: int) -> None:
85+
url, responses, requests = endpoint
86+
responses.extend([(status, "temporarily unavailable"), (200, '{"choices":[{}]}')])
87+
client.verify_endpoint(url, "model")
88+
assert len(requests) == 2
89+
90+
91+
def test_heartbeat_request_remains_best_effort(client: ModuleType, endpoint) -> None:
92+
url, responses, requests = endpoint
93+
responses.append((400, "invalid request"))
94+
assert client.post_one_token(url, "model", timeout=1) is None
95+
assert len(requests) == 1
96+
97+
98+
def test_transport_failure_remains_retryable(client: ModuleType, monkeypatch) -> None:
99+
def unavailable(*args: object, **kwargs: object) -> None:
100+
raise urllib.error.URLError("connection refused")
101+
102+
monkeypatch.setattr(client.urllib.request, "urlopen", unavailable)
103+
assert client.post_one_token("http://unused", "model", timeout=1) is None
104+
105+
106+
def test_rejection_body_is_bounded(client: ModuleType, endpoint) -> None:
107+
url, responses, _ = endpoint
108+
responses.append((400, "x" * 10000))
109+
with pytest.raises(RuntimeError, match="HTTP 400") as error:
110+
client.verify_endpoint(url, "model")
111+
assert len(str(error.value)) < 2300

0 commit comments

Comments
 (0)