From a2d36fbb0e5808194af21101ca548f17bdd759ec Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 22 Sep 2026 10:03:30 -0500 Subject: [PATCH 1/2] fix(trtllm): forward served model name to native workers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将显式配置的服务模型名传给原生 TRT-LLM worker，确保服务器与 benchmark/eval 客户端使用一致的 API 模型名。 Signed-off-by: adibarra <93070681+adibarra@users.noreply.github.com> --- docs/config-reference.md | 5 +++ examples/trtllm/trtllm-serve-agg.yaml | 1 + src/srtctl/backends/trtllm.py | 2 ++ tests/test_trtllm_served_model_name.py | 45 ++++++++++++++++++++++++++ 4 files changed, 53 insertions(+) diff --git a/docs/config-reference.md b/docs/config-reference.md index e2c7d847a..66a01d240 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -260,6 +260,11 @@ engine: served_model_name: "Qwen/Qwen3-0.6B" ``` +For `trtllm_serve`, an explicit `served_model_name` is passed to the worker's +`--served_model_name` option so the server and benchmark/eval clients use the same +API model name. Leave it unset to retain the server's default. This is a server +option, not a key in `roles.*.args` (the engine YAML). + ```yaml engine: type: mocker diff --git a/examples/trtllm/trtllm-serve-agg.yaml b/examples/trtllm/trtllm-serve-agg.yaml index 7f15c3851..22c14fc53 100644 --- a/examples/trtllm/trtllm-serve-agg.yaml +++ b/examples/trtllm/trtllm-serve-agg.yaml @@ -26,6 +26,7 @@ frontend: engine: type: trtllm + # Also sets trtllm-serve's API model name; no duplicate extra_args flag is needed. served_model_name: "Qwen/Qwen3-0.6B" roles: agg: diff --git a/src/srtctl/backends/trtllm.py b/src/srtctl/backends/trtllm.py index 9f8d974cc..d01009fe1 100644 --- a/src/srtctl/backends/trtllm.py +++ b/src/srtctl/backends/trtllm.py @@ -394,6 +394,8 @@ def build_worker_command( # ai-dynamo tensorrtllm-runtime 1.3.0-dev.1 container, which accept --config; # some trtllm-serve builds spell this --extra_llm_api_options. cmd.extend(["--config", str(container_config_path)]) + if self.served_model_name: + cmd.extend(["--served_model_name", self.served_model_name]) cmd.extend(self.get_extra_args_for_mode(mode)) return self._wrap_with_numa_cpu_bind(cmd) diff --git a/tests/test_trtllm_served_model_name.py b/tests/test_trtllm_served_model_name.py index d003ccb47..194a26413 100644 --- a/tests/test_trtllm_served_model_name.py +++ b/tests/test_trtllm_served_model_name.py @@ -3,7 +3,15 @@ """Tests for TRT-LLM's served model name override.""" +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + from srtctl.backends.trtllm import TRTLLMProtocol, TRTLLMServerConfig +from srtctl.core.schema import DynamoConfig +from srtctl.core.topology import Process class TestTRTLLMServedModelName: @@ -53,3 +61,40 @@ def test_reaches_the_worker_command(self): assert "--served-model-name" in cmd assert cmd[cmd.index("--served-model-name") + 1] == "deepseek-ai/deepseek-r1" + + @pytest.mark.parametrize("mode,name", [("agg", "org/model"), ("prefill", "org/model"), ("agg", None)]) + def test_direct_worker_receives_explicit_name(self, tmp_path, mode, name): + backend = TRTLLMProtocol(served_model_name=name) + runtime = SimpleNamespace( + model_path=Path("/weights/checkpoint"), + worker_model_arg="/model", + log_dir=tmp_path, + gpu_type="h200", + frontend_port=8000, + dynamo=DynamoConfig(), + ) + process = Process( + node="worker", + gpu_indices=frozenset({0}), + sys_port=7500, + http_port=9001, + endpoint_mode=mode, + endpoint_index=0, + ) + command = backend.build_worker_command(process, [process], runtime, frontend_type="trtllm_serve") + + expected = [ + "trtllm-llmapi-launch", + "trtllm-serve", + "/model", + "--host", + "0.0.0.0", + "--port", + "8000" if mode == "agg" else "9001", + "--config", + f"/logs/trtllm_config_{mode}.yaml", + ] + if name is not None: + expected += ["--served_model_name", "org/model"] + assert command == expected + assert yaml.safe_load((tmp_path / f"trtllm_config_{mode}.yaml").read_text()) == {} From ed9fab3e2bef839152f1fdad265faa0696eb0669 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:35:52 -0500 Subject: [PATCH 2/2] test(trtllm): focus served model name regression coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按现有测试风格只验证模型名参数的传递和省略，去掉无关命令细节及重复路径断言，并简化示例注释。 Signed-off-by: adibarra <93070681+adibarra@users.noreply.github.com> --- examples/trtllm/trtllm-serve-agg.yaml | 2 +- tests/test_trtllm_served_model_name.py | 30 +++++++------------------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/examples/trtllm/trtllm-serve-agg.yaml b/examples/trtllm/trtllm-serve-agg.yaml index 22c14fc53..373b22897 100644 --- a/examples/trtllm/trtllm-serve-agg.yaml +++ b/examples/trtllm/trtllm-serve-agg.yaml @@ -26,7 +26,7 @@ frontend: engine: type: trtllm - # Also sets trtllm-serve's API model name; no duplicate extra_args flag is needed. + # API model name shared by the server and benchmark/eval clients. served_model_name: "Qwen/Qwen3-0.6B" roles: agg: diff --git a/tests/test_trtllm_served_model_name.py b/tests/test_trtllm_served_model_name.py index 194a26413..6f5ba6fbd 100644 --- a/tests/test_trtllm_served_model_name.py +++ b/tests/test_trtllm_served_model_name.py @@ -5,9 +5,9 @@ from pathlib import Path from types import SimpleNamespace +from unittest.mock import MagicMock import pytest -import yaml from srtctl.backends.trtllm import TRTLLMProtocol, TRTLLMServerConfig from srtctl.core.schema import DynamoConfig @@ -47,9 +47,6 @@ def test_is_not_written_into_the_engine_yaml(self): def test_reaches_the_worker_command(self): """The worker must actually be launched with the configured name.""" - from pathlib import Path - from unittest.mock import MagicMock - backend = TRTLLMProtocol(served_model_name="deepseek-ai/deepseek-r1") runtime = MagicMock() runtime.model_path = Path("/models/deepseek_r1-torch-fp4-v2") @@ -62,8 +59,8 @@ def test_reaches_the_worker_command(self): assert "--served-model-name" in cmd assert cmd[cmd.index("--served-model-name") + 1] == "deepseek-ai/deepseek-r1" - @pytest.mark.parametrize("mode,name", [("agg", "org/model"), ("prefill", "org/model"), ("agg", None)]) - def test_direct_worker_receives_explicit_name(self, tmp_path, mode, name): + @pytest.mark.parametrize("name", ["org/model", None]) + def test_direct_worker_receives_explicit_name(self, tmp_path: Path, name: str | None) -> None: backend = TRTLLMProtocol(served_model_name=name) runtime = SimpleNamespace( model_path=Path("/weights/checkpoint"), @@ -78,23 +75,12 @@ def test_direct_worker_receives_explicit_name(self, tmp_path, mode, name): gpu_indices=frozenset({0}), sys_port=7500, http_port=9001, - endpoint_mode=mode, + endpoint_mode="agg", endpoint_index=0, ) command = backend.build_worker_command(process, [process], runtime, frontend_type="trtllm_serve") - expected = [ - "trtllm-llmapi-launch", - "trtllm-serve", - "/model", - "--host", - "0.0.0.0", - "--port", - "8000" if mode == "agg" else "9001", - "--config", - f"/logs/trtllm_config_{mode}.yaml", - ] - if name is not None: - expected += ["--served_model_name", "org/model"] - assert command == expected - assert yaml.safe_load((tmp_path / f"trtllm_config_{mode}.yaml").read_text()) == {} + if name is None: + assert "--served_model_name" not in command + else: + assert command[command.index("--served_model_name") + 1] == "org/model"