From 8420094e2782c7bdca8c7baab42b4a99d159477c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 5 Oct 2026 18:33:36 -0500 Subject: [PATCH 01/25] feat(frontends): add the llm-d router frontend.type: llm-d runs llm-d's Endpoint Picker (EPP) behind Envoy in front of direct vLLM workers. llm-d discovers workers through Kubernetes or a file, so srtctl is the registrar: once every worker answers /health it writes the EPP's file-discovery endpoints, the EPP configuration (the recipe's new frontend.epp_config plus the discovery plugin), and the Envoy configuration into the log directory, then starts the EPP and Envoy on the router node. The job is ready when Envoy's admin /ready answers and the EPP's llm_d_epp_ready_endpoints gauge counts every endpoint. For prefill/decode the frontend implies llm-d's P/D sidecar as a per-worker service (type llm-d-sidecar) on every routable decode worker, started before the workers and gated on its /health. The EPP addresses the sidecar, which runs the prefill and hands the KV transfer parameters to its vLLM. Its listener is a new allocator kind, PROXY_PORTS, carried on Process.proxy_port: a frontend names the modes it reaches through a proxy (Frontend.proxied_worker_modes) and the job topology allocates the port for their routable workers; a service kind attaches its per-worker instances only where it applies (ServiceKind.attaches_to). Per-worker services also get {worker_http_port} and {worker_proxy_port}. The sidecar's --kv-connector protocol comes from the decode workers' vLLM connector class (VLLMBackend.kv_connector_class); NixlConnector maps to nixlv2. Signed-off-by: Cam Quilici --- docs/frontends.md | 4 +- docs/llm-d.md | 162 +++++++++ docs/schema-reference.md | 3 +- docs/schema/recipe.schema.json | 14 +- docs/services.md | 6 +- examples/README.md | 3 +- examples/vllm/llm-d-agg.yaml | 63 ++++ examples/vllm/llm-d-disagg.yaml | 94 +++++ mkdocs.yml | 1 + src/srtctl/backends/vllm.py | 26 ++ src/srtctl/cli/mixins/service_stage.py | 5 +- src/srtctl/core/AGENTS.md | 2 +- src/srtctl/core/schema.py | 11 +- src/srtctl/core/topology.py | 3 + src/srtctl/core/worker_backends.py | 33 +- src/srtctl/frontends/AGENTS.md | 4 +- src/srtctl/frontends/__init__.py | 3 + src/srtctl/frontends/base.py | 9 + src/srtctl/frontends/llm_d.py | 335 ++++++++++++++++++ src/srtctl/frontends/static_router.py | 68 ++-- src/srtctl/ports.py | 14 + src/srtctl/services/__init__.py | 13 +- src/srtctl/services/llm_d_sidecar.py | 115 ++++++ src/srtctl/services/registry.py | 15 + src/srtctl/templates/llm-d-envoy.yaml.j2 | 94 +++++ tests/snapshots/launch/vllm__llm-d-agg.txt | 190 ++++++++++ tests/snapshots/launch/vllm__llm-d-disagg.txt | 205 +++++++++++ tests/test_e2e.py | 2 +- tests/test_frontends.py | 3 +- tests/test_llm_d_frontend.py | 315 ++++++++++++++++ tests/test_port_allocator.py | 3 + tests/test_services.py | 1 + 32 files changed, 1768 insertions(+), 51 deletions(-) create mode 100644 docs/llm-d.md create mode 100644 examples/vllm/llm-d-agg.yaml create mode 100644 examples/vllm/llm-d-disagg.yaml create mode 100644 src/srtctl/frontends/llm_d.py create mode 100644 src/srtctl/services/llm_d_sidecar.py create mode 100644 src/srtctl/templates/llm-d-envoy.yaml.j2 create mode 100644 tests/snapshots/launch/vllm__llm-d-agg.txt create mode 100644 tests/snapshots/launch/vllm__llm-d-disagg.txt create mode 100644 tests/test_llm_d_frontend.py diff --git a/docs/frontends.md b/docs/frontends.md index f962b2b4d..865607ebf 100644 --- a/docs/frontends.md +++ b/docs/frontends.md @@ -1,6 +1,6 @@ # Frontends and Dynamo -The `frontend:` block (which router fronts the workers) and the `dynamo:` block (how Dynamo is installed and wired). Router-specific pages: [SGLang Router](sglang-router.md), [vLLM Router](vllm-router.md), [Shepherd Model Gateway](smg.md). +The `frontend:` block (which router fronts the workers) and the `dynamo:` block (how Dynamo is installed and wired). Router-specific pages: [SGLang Router](sglang-router.md), [vLLM Router](vllm-router.md), [Shepherd Model Gateway](smg.md), [llm-d Router](llm-d.md). ## frontend @@ -8,7 +8,7 @@ Frontend/router configuration. ```yaml frontend: - # Frontend type: "dynamo" (default), "sglang-router", "vllm-router", "smg", direct "sglang", "vllm", "trtllm_serve", + # Frontend type: "dynamo" (default), "sglang-router", "vllm-router", "smg", "llm-d", direct "sglang", "vllm", "trtllm_serve", # or "none" for a services-only job (no router, no OpenAI endpoint, no worker-count health gate; only # valid without engine roles, see services[].nodes) type: dynamo diff --git a/docs/llm-d.md b/docs/llm-d.md new file mode 100644 index 000000000..4d96c81fc --- /dev/null +++ b/docs/llm-d.md @@ -0,0 +1,162 @@ +# llm-d router + +`frontend.type: llm-d` routes requests with the [llm-d](https://llm-d.ai) router: the +Endpoint Picker (EPP) behind Envoy, in front of direct `vllm serve` workers. The EPP +scores the workers with the plugins in `frontend.epp_config` (prefix-cache affinity, +queue depth, load, ...) and, for prefill/decode, picks a prefill and a decode worker for +every request. No Dynamo, NATS, or etcd is involved. + +Upstream references are pinned to llm-d-router +[v0.11.0](https://github.com/llm-d/llm-d-router/tree/a5cbe600ebade00cf3e9885beaf2bfacddeabce1) +and the llm-d [no-Kubernetes guide](https://github.com/llm-d/llm-d/tree/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment). + +## What runs where + +| Process | Where | Launched as | Port | +| --- | --- | --- | --- | +| Envoy | router node | the frontend (`llm-d-envoy_0`) | the public port (8000); admin `/ready` on 9901 | +| EPP | router node | the frontend (`llm-d-epp_0`) | ext_proc gRPC 9002, gRPC health 9003, Prometheus 9090 | +| P/D sidecar | every routable decode worker (prefill/decode jobs only) | the implied `llm-d-sidecar` service | `Process.proxy_port`, from the allocator | +| vLLM | every worker | the backend, as for the other static routers | `Process.http_port` | + +Envoy owns the public port. For every request it asks the EPP over ext_proc +(`FULL_DUPLEX_STREAMED`) which endpoint to use; the EPP sets the +`x-gateway-destination-endpoint` header and Envoy's `ORIGINAL_DST` cluster forwards the +request there. For prefill/decode the EPP picks a decode endpoint and names the chosen +prefill worker in `x-prefiller-host-port` +([`disagg_profile_handler.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/epp/framework/plugins/scheduling/profilehandler/disagg/disagg_profile_handler.go#L533-L536)). +The decode endpoint is llm-d's P/D sidecar: it sends the request to that prefill worker +first, then hands the returned `kv_transfer_params` to its own vLLM, whose KV connector +pulls the cache from the prefill worker +([`connector_nixlv2.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/sidecar/proxy/connector_nixlv2.go#L42)). +Everything else, `/metrics` included, the sidecar proxies to its vLLM. + +### Why the EPP and Envoy are the frontend and the sidecar is a service + +The frontend is the process that owns the public port and decides when the job is +ready, so Envoy and the EPP are the frontend. They also have to start at the frontend's +moment: llm-d discovers workers through Kubernetes or a file, and the workers here are +plain engines that register nowhere, so srtctl is the registrar. It writes the endpoints +file once every worker answers `/health` (the static routers' pre-start gate), then starts +the EPP and Envoy; no service phase sits between the workers and the frontend. + +The sidecar is a different process with its own image, placement, and health check, one +per decode worker, on the worker's node; that is what `services` with +`placement.per: worker` is for. Its `GET /health` answers 200 on its own +([`proxy.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/sidecar/proxy/proxy.go#L586-L588)), +so it starts in the `before_workers` phase and gates on that probe. It is critical: a +dead sidecar takes its decode worker out of the routing pool. The frontend implies it, +as Dynamo implies etcd; declare a service named `llm-d-sidecar` to change it: + +```yaml +services: + - name: llm-d-sidecar + type: llm-d-sidecar + container: my-llm-d-image # default: frontend.container_image, else the job container + args: + - --enable-prefiller-sampling # appended after the flags srtctl sets +``` + +srtctl sets `--port` (the worker's `Process.proxy_port`), `--model-server-port` (its +`Process.http_port`), `--kv-connector`, and `--secure-proxy=false`; `args` come last. +The allocator hands out the proxy port (`PROXY_PORTS`) for every routable worker of a +mode the frontend proxies (`Frontend.proxied_worker_modes`), so two decode workers on +one node never collide. The `--kv-connector` protocol comes from the decode workers' vLLM +connector class (`VLLMBackend.kv_connector_class`, which reads a role's own +`kv-transfer-config` before `connector`): + +| Decode connector | Sidecar protocol | +| --- | --- | +| `NixlConnector` | `nixlv2` | + +Any other decode connector is rejected at load time. + +## Endpoint discovery + +The EPP runs without Kubernetes when its configuration names a discovery plugin +([`runner.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/cmd/epp/runner/runner.go#L264-L273), +[`runWithFileDiscovery`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/cmd/epp/runner/runner.go#L1044-L1196)). +srtctl writes three files into the log directory (every container sees it at `/logs`): + +- `llm-d-endpoints.yaml`: one endpoint per routable worker process (the one with an HTTP + port: a multi-node worker's leader), at its IPv4 address on the cluster's network + interface, labelled `llm-d.ai/role: prefill`, `decode`, or `both` (aggregate). A decode + endpoint is its sidecar's port. +- `llm-d-epp-config.yaml`: `frontend.epp_config` with the + [`file-discovery`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/epp/framework/plugins/datalayer/discovery/file/plugin.go) + plugin (`watchFile: false`) and `dataLayer.discovery` added. `discovery.pluginRef` is + the spelling v0.10 and v0.11 both accept. A recipe that configures discovery itself is + rejected. +- `llm-d-envoy.yaml`: the guide's Envoy configuration with the public port, an admin + listener srtctl can reach, no request timeout, and an access log + (`llm-d-envoy-access.log`) recording, for every request, the endpoint the EPP picked + (`upstream=`) and the prefill worker it named (`prefiller=`). + +## Readiness and metrics + +The job is ready when Envoy's admin `/ready` answers 200 and the EPP's +`llm_d_epp_ready_endpoints` gauge, the endpoints whose metrics it scraped within its +staleness window +([`llm_d_router_metrics.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/epp/metrics/llm_d_router_metrics.go#L234-L241), +[`logger.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/epp/datalayer/logger/logger.go#L106-L117)), +counts every endpoint in the file. The EPP scrapes a worker on the address it routes +to, so a decode worker is scraped through its sidecar. Tachometer scrapes the EPP's +Prometheus listener as the frontend; worker metrics stay on vLLM's own port. + +## Images + +The EPP, Envoy, and the sidecar run from `frontend.container_image` (the sidecar unless +its service sets `container`), or the job container when it is unset, as the executables +`epp`, `envoy`, and `pd-sidecar` on `PATH`. llm-d publishes the EPP and the sidecar as +distroless images (`/app/epp`, `/app/pd-sidecar`), which pyxis cannot start: it enters a +container through `sh` +([`pyxis_slurmstepd.c`](https://github.com/NVIDIA/pyxis/blob/107519944221822ea1dace4db8e7234b2eaa4cd5/pyxis_slurmstepd.c#L1128-L1135)). Build an image with a shell that carries the three executables, +for example on top of the vLLM image the workers run: + + +```dockerfile +FROM vllm/vllm-openai: +COPY --from=ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.11.0 /app/epp /usr/local/bin/epp +COPY --from=ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.11.0 /app/pd-sidecar /usr/local/bin/pd-sidecar +COPY --from=envoyproxy/envoy:distroless-v1.33.2 /usr/local/bin/envoy /usr/local/bin/envoy +``` + +## Recipe + +```yaml +frontend: + type: llm-d + enable_multiple_frontends: false + container_image: llm-d-vllm # optional: the router image + epp_config: # EndpointPickerConfig without discovery + plugins: [...] + schedulingProfiles: [...] + args: # extra EPP flags, e.g. log verbosity + v: 2 +engine: + type: vllm + connector: nixl +``` + +`frontend.args` are passed to the EPP; the pool, configuration, and port flags are +srtctl's. Prefill/decode needs `epp_config` with `prefill` and `decode` scheduling +profiles and the `disagg-profile-handler`; an aggregate job without `epp_config` runs +the EPP's built-in default profile. See +[`examples/vllm/llm-d-disagg.yaml`](https://github.com/NVIDIA/srt-slurm/blob/main/examples/vllm/llm-d-disagg.yaml) and +[`examples/vllm/llm-d-agg.yaml`](https://github.com/NVIDIA/srt-slurm/blob/main/examples/vllm/llm-d-agg.yaml). + +A vLLM worker with `data-parallel-size` is one `vllm serve` that balances its DP ranks +itself (the leader serves the API, other nodes run headless ranks), so it is one EPP +endpoint, as it is one SMG worker. + +## Limitations + +- vLLM workers over HTTP only; gRPC roles are rejected. +- One router replica: the EPP keeps its scheduling state (the approximate prefix index, + in-flight load) in memory, so `enable_multiple_frontends` with more than one router is + rejected. +- The endpoints file is written once (`watchFile: false`) and the EPP does not eject a + worker that fails; a crashed worker or sidecar fails the job instead. +- The EPP routes to a worker, not to one of its DP ranks. +- Precise prefix-cache routing (KV events into the EPP) is not wired up; use the + approximate prefix scorers. diff --git a/docs/schema-reference.md b/docs/schema-reference.md index 717531540..910aa3117 100644 --- a/docs/schema-reference.md +++ b/docs/schema-reference.md @@ -121,7 +121,7 @@ Frontend/router configuration. | Key | Type | Default | Description | |---|---|---|---| -| `type` | str | `'dynamo'` | Frontend type - "dynamo" (default); "sglang-router" (SGLang Model Gateway), "vllm-router", "smg" (Shepherd Model Gateway, any backend), "atomesh", and "tilert-router" (static routers); "sglang", "vllm", and "trtllm_serve" (direct: the single aggregate worker binds the public port, no router process); "none" (services-only job: no router, no OpenAI endpoint, no worker-count health gate; requires no engine roles). Pre-2.0 recipes spelled the router "sglang"; ``srtctl migrate`` rewrites that to "sglang-router". | +| `type` | str | `'dynamo'` | Frontend type - "dynamo" (default); "sglang-router" (SGLang Model Gateway), "vllm-router", "smg" (Shepherd Model Gateway, any backend), "llm-d" (llm-d Endpoint Picker behind Envoy, vLLM), "atomesh", and "tilert-router" (static routers); "sglang", "vllm", and "trtllm_serve" (direct: the single aggregate worker binds the public port, no router process); "none" (services-only job: no router, no OpenAI endpoint, no worker-count health gate; requires no engine roles). Pre-2.0 recipes spelled the router "sglang"; ``srtctl migrate`` rewrites that to "sglang-router". | | `enable_multiple_frontends` | bool | `True` | Scale with nginx + multiple routers. When ``True`` (default), srtctl stands up nginx and fans out to ``num_additional_frontends + 1`` router replicas. When ``False``, there is NO nginx proxy — the benchmark must target the single master router (or a worker) directly at ``http://localhost:``. ``benchmark.command`` has no placeholder substitution, so write the URL out literally. | | `num_additional_frontends` | int | `9` | Additional routers beyond master (default: 9) | | `nginx_container` | str | `'nginx:1.27.4'` | Custom nginx container image (default: nginx:1.27.4) | @@ -130,6 +130,7 @@ Frontend/router configuration. | `nginx_session_affinity_header` | str | `'X-Dynamo-Session-ID'` | Header hashed when affinity is on (default ``X-Dynamo-Session-ID``). Set ``X-Correlation-ID`` for clients (e.g. aiperf) that carry the session id in that header instead. | | `nginx_keepalive_timeout` | str | `'600s'` | Idle timeout for client and upstream keepalive connections in the generated nginx.conf (default "600s"). nginx's own default is 75s, which closes a session's connection during the long recorded think-time of an agentic replay; the client's next write on that pooled socket then fails with "broken pipe" / "server disconnected" and nothing is logged server-side. | | `worker_selection` | dict[str, Any] \| None | `None` | Inline Dynamo worker-selection policy configuration. srtctl writes this mapping under the top-level ``worker_selection`` key in a generated router policy YAML and passes it to the Dynamo frontend via ``--router-policy-config``. | +| `epp_config` | dict[str, Any] \| None | `None` | llm-d Endpoint Picker configuration (``EndpointPickerConfig``: ``plugins``, ``schedulingProfiles``, ...) for ``frontend.type: llm-d``. srtctl writes it to a file with the ``file-discovery`` plugin and ``dataLayer.discovery`` added, so the EPP reads the job's workers from the endpoints file srtctl renders. Required for prefill/decode jobs (the scheduler's P/D profiles); omitted, an aggregate job runs the EPP's built-in default profile. | | `args` | dict[str, Any] \| None | `None` | CLI arguments passed to the frontend/router process | | `env` | dict[str, str] \| None | `None` | Environment variables for frontend processes | | `container_image` | str \| None | `None` | Optional router-specific image. Static routers use the model/backend image when omitted. | diff --git a/docs/schema/recipe.schema.json b/docs/schema/recipe.schema.json index 3a46a60d8..d3288f824 100644 --- a/docs/schema/recipe.schema.json +++ b/docs/schema/recipe.schema.json @@ -868,7 +868,7 @@ "properties": { "type": { "type": "string", - "description": "Frontend type - \"dynamo\" (default); \"sglang-router\" (SGLang Model Gateway), \"vllm-router\", \"smg\" (Shepherd Model Gateway, any backend), \"atomesh\", and \"tilert-router\" (static routers); \"sglang\", \"vllm\", and \"trtllm_serve\" (direct: the single aggregate worker binds the public port, no router process); \"none\" (services-only job: no router, no OpenAI endpoint, no worker-count health gate; requires no engine roles). Pre-2.0 recipes spelled the router \"sglang\"; ``srtctl migrate`` rewrites that to \"sglang-router\".", + "description": "Frontend type - \"dynamo\" (default); \"sglang-router\" (SGLang Model Gateway), \"vllm-router\", \"smg\" (Shepherd Model Gateway, any backend), \"llm-d\" (llm-d Endpoint Picker behind Envoy, vLLM), \"atomesh\", and \"tilert-router\" (static routers); \"sglang\", \"vllm\", and \"trtllm_serve\" (direct: the single aggregate worker binds the public port, no router process); \"none\" (services-only job: no router, no OpenAI endpoint, no worker-count health gate; requires no engine roles). Pre-2.0 recipes spelled the router \"sglang\"; ``srtctl migrate`` rewrites that to \"sglang-router\".", "default": "dynamo" }, "enable_multiple_frontends": { @@ -918,6 +918,18 @@ "description": "Inline Dynamo worker-selection policy configuration. srtctl writes this mapping under the top-level ``worker_selection`` key in a generated router policy YAML and passes it to the Dynamo frontend via ``--router-policy-config``.", "default": null }, + "epp_config": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "description": "llm-d Endpoint Picker configuration (``EndpointPickerConfig``: ``plugins``, ``schedulingProfiles``, ...) for ``frontend.type: llm-d``. srtctl writes it to a file with the ``file-discovery`` plugin and ``dataLayer.discovery`` added, so the EPP reads the job's workers from the endpoints file srtctl renders. Required for prefill/decode jobs (the scheduler's P/D profiles); omitted, an aggregate job runs the EPP's built-in default profile.", + "default": null + }, "args": { "anyOf": [ { diff --git a/docs/services.md b/docs/services.md index 4b4b76bb5..0105ba292 100644 --- a/docs/services.md +++ b/docs/services.md @@ -48,7 +48,7 @@ directory. `examples/features/services.yaml` is a runnable version of this. ```yaml services: - name: my-sidecar # required, unique across the list - type: generic # generic (default) | etcd | nats | mooncake-master | dcgm-exporter | node-exporter | mooncake-store | lmcache-server + type: generic # generic (default) | etcd | nats | mooncake-master | dcgm-exporter | node-exporter | mooncake-store | lmcache-server | llm-d-sidecar enabled: true # false drops the service (the way to switch an implied one off) external: null # typed kinds only: use an instance that already runs at this address command: # argv, not shell-interpreted; required for generic @@ -135,6 +135,7 @@ Things the recipe asks for elsewhere are services the job runs without an entry; | a declared `mooncake-master` entry (see [Mooncake KV Store](mooncake-kv-store.md)) | `mooncake-master` | the infra node, phase `before_workers` | | a vLLM role with `connector: lmcache-mp` (engine-level or `roles..args.connector`), unless a `lmcache-server` entry is declared | `lmcache-server` | that role's nodes, or every worker node when several roles use it, phase `before_workers` | | `engine.failover` (see [Shadow Engine Recovery](shadow-engine-recovery.md)) | `gms` | one instance per vLLM worker (`placement.per: worker`), phase `before_workers` | +| `frontend.type: llm-d` with prefill and decode workers (see [llm-d Router](llm-d.md)) | `llm-d-sidecar` | one instance per routable decode worker (`placement.per: worker`), phase `before_workers` | | tachometer on (the default; `observability.tachometer.enabled`) | `dcgm-exporter`, `node-exporter` | every worker node, phase `after_frontend` | `srtctl dry-run` lists them next to the declared ones, marked `implied by:`. A declared entry with @@ -191,7 +192,7 @@ node are rejected before anything launches; give them disjoint placements or por of the placed role(s) on each selected node (`node` must be `prefill`, `decode`, `agg`, or `workers`), a sidecar in the Kubernetes sense. The instance runs in that worker's device view (its `CUDA_VISIBLE_DEVICES`, the same pinning the worker's engine steps get) and sees `{worker_role}`, -`{worker_index}`, `{worker_node_rank}`, `{worker_gpus}` and `{worker_gpu_count}`. Its step and log are +`{worker_index}`, `{worker_node_rank}`, `{worker_gpus}` and `{worker_gpu_count}`, plus `{worker_http_port}` when the worker serves an API and `{worker_proxy_port}` when the frontend reaches it through a proxy (`Process.proxy_port`). The kind decides which workers get an instance: every one by default; the `llm-d-sidecar` only the routable decode workers it fronts. Its step and log are `service____`. Because the instances of one node share its network namespace, a per-worker service's `readiness` must be a log probe. The GPU Memory Service of [shadow engine recovery](shadow-engine-recovery.md) is the built-in per-worker kind, and [SGLang fast engine recovery](sglang-weight-cache.md) runs SGLang's weight cache daemon as a plain `generic` per-worker service: @@ -305,6 +306,7 @@ environment its process needs; the launch path is shared by every kind. Register | `mooncake-store` | `python -m mooncake.mooncake_store_service` | `before_workers` | `true` | Requires a `mooncake-master` entry. Container falls back to the master's. Injects the master's address. | | `lmcache-server` | `lmcache server` bound to the RPC and HTTP ports srtctl owns (8750, 8751) | `before_workers` | `true` | One per worker node (`placement.node: workers`); vLLM ranks reach it over localhost with `connector: lmcache-mp`, which implies it. SGLang workers started with `enable-lmcache` get `LMCACHE_MP_HOST`/`LMCACHE_MP_PORT` pointing at it unless the recipe sets `lmcache-config-file` or `LMCACHE_MP_HOST`. Services do not inherit the top-level `environment:`, so set `PYTHONHASHSEED` in its `env:` when the workers set it. `args` are appended (`--l1-size-gb`, `--chunk-size`, `--max-workers`, ...). Ready when `GET /healthcheck` answers; LMCache must be installed in the job container. | | `ray` | `ray start --head ...` on the first instance, `ray start --address=:6379 ...` on the rest, both `--block` | `before_workers` | `true` | One Ray cluster across the service's nodes; see [Ray cluster](#ray-cluster). Placement `workers` (default), `head`, or `all`. `options`: `port` (GCS, 6379), `dashboard_port` (8265), `num_gpus` (the node's count). `args` are appended to every `ray start`. | +| `llm-d-sidecar` | `pd-sidecar --port= --model-server-port= --kv-connector= --secure-proxy=false` in `frontend.container_image`, else the job container | `before_workers` | `true` | Implied by `frontend.type: llm-d` for prefill/decode; see [llm-d Router](llm-d.md). `placement.per: worker` on decode: one instance per routable decode worker, bound to its allocated `Process.proxy_port`. `command` replaces the executable, `args` are appended. Ready when `GET /health` on its port answers. | | `gms` | one `python3 -m gpu_memory_service --device k` per GPU of the worker, supervised by bash, in the job container | `before_workers` | `true` | Implied by `engine.failover`; see [Shadow Engine Recovery](shadow-engine-recovery.md). `placement.per: worker` (required): one instance per vLLM worker in that worker's device view, sockets and lock file under `/srtctl-/_`. Ready when its log says `GMS ready:`; `readiness.timeout_seconds` (default 120) also bounds the servers' startup. | ### Ray cluster diff --git a/examples/README.md b/examples/README.md index 8b7bcb0b1..dd925a792 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,7 +7,7 @@ Small starting points, one per frontend and topology. The matrix examples serve | Backend | Dynamo frontend | Native router | Router-free direct | | --- | --- | --- | --- | | SGLang | `sglang/dynamo-agg.yaml`, `sglang/dynamo-disagg.yaml` | `sglang/sglang-router-agg.yaml`, `sglang/sglang-router-disagg.yaml`, `sglang/smg-disagg.yaml` (SMG) | `sglang/sglang-direct-agg.yaml` | -| vLLM | `vllm/dynamo-agg.yaml`, `vllm/dynamo-disagg.yaml` | `vllm/vllm-router-agg.yaml`, `vllm/vllm-router-disagg.yaml`, `vllm/vllm-router-moriio-disagg.yaml` (ROCm, MoRI-IO discovery), `vllm/smg-agg.yaml`, `vllm/smg-dep16.yaml`, `vllm/smg-disagg-grpc.yaml` (SMG; P/D over gRPC with NIXL) | `vllm/vllm-direct-agg.yaml` | +| vLLM | `vllm/dynamo-agg.yaml`, `vllm/dynamo-disagg.yaml` | `vllm/vllm-router-agg.yaml`, `vllm/vllm-router-disagg.yaml`, `vllm/vllm-router-moriio-disagg.yaml` (ROCm, MoRI-IO discovery), `vllm/smg-agg.yaml`, `vllm/smg-dep16.yaml`, `vllm/smg-disagg-grpc.yaml` (SMG; P/D over gRPC with NIXL), `vllm/llm-d-agg.yaml`, `vllm/llm-d-disagg.yaml` (llm-d EPP behind Envoy; P/D through the llm-d sidecar) | `vllm/vllm-direct-agg.yaml` | | TRT-LLM | `trtllm/dynamo-agg.yaml`, `trtllm/dynamo-disagg.yaml`, `trtllm/dynamo-disagg-multinode.yaml` (TP16 decode across two 8-GPU nodes: one srun, one host per rank under `--distribution=arbitrary`) | `trtllm/trtllm-serve-disagg.yaml`, `trtllm/smg-agg.yaml` (SMG) | `trtllm/trtllm-serve-agg.yaml` | | TokenSpeed | `tokenspeed/dynamo-agg.yaml`, `tokenspeed/dynamo-disagg.yaml` | `tokenspeed/smg-agg.yaml`, `tokenspeed/smg-disagg.yaml` (SMG, gRPC engines) | | | Mocker | `mocker/dynamo-agg.yaml` | | | @@ -63,6 +63,7 @@ containers: vllm-lmcache: /path/to/vllm-lmcache.sqsh # vLLM image with LMCache installed, for features/lmcache-server.yaml and -disagg.yaml sglang-lmcache: /path/to/sglang-lmcache.sqsh # SGLang image with LMCache installed, for features/lmcache-server-sglang.yaml smg: /path/to/smg.sqsh # Shepherd Model Gateway image (lightseekorg/smg), the router for the smg examples + llm-d-vllm: /path/to/llm-d-vllm.sqsh # vLLM image with epp, pd-sidecar and envoy on PATH, for the llm-d examples (docs/llm-d.md#images) ``` ## Optional GPU temperature diff --git a/examples/vllm/llm-d-agg.yaml b/examples/vllm/llm-d-agg.yaml new file mode 100644 index 000000000..fc69f0589 --- /dev/null +++ b/examples/vllm/llm-d-agg.yaml @@ -0,0 +1,63 @@ +# Two aggregated vLLM workers behind the llm-d router (docs/llm-d.md). +# +# On the router node srtctl starts llm-d's Endpoint Picker (EPP) and Envoy on the +# public port; the EPP reads both workers from an endpoints file srtctl writes and +# picks one per request with the scorers below. +# +# Aliases (`qwen3-0.6b`, `llm-d-vllm`) resolve through srtslurm.yaml; see +# examples/README.md. `llm-d-vllm` is a vLLM image that also ships the `epp`, +# `pd-sidecar`, and `envoy` executables on PATH (docs/llm-d.md#images). + +schema: 2 +name: "qwen3-0.6b-vllm-llm-d-agg" + +slurm: + time_limit: "00:30:00" + +model: + path: "qwen3-0.6b" + container: "llm-d-vllm" + precision: "bf16" + +resources: + gpu_type: "h100" + gpus_per_node: 8 +frontend: + type: llm-d + enable_multiple_frontends: false + # Prefix-cache affinity weighted over queue depth; srtctl adds the file-discovery plugin. + epp_config: + plugins: + # Hashes each prompt's blocks into the prefix index the prefix-cache scorer reads. + - type: approx-prefix-cache-producer + - type: prefix-cache-scorer + - type: queue-scorer + - type: max-score-picker + schedulingProfiles: + - name: default + plugins: + - pluginRef: prefix-cache-scorer + weight: 3 + - pluginRef: queue-scorer + weight: 2 + - pluginRef: max-score-picker + +engine: vllm +roles: + agg: + nodes: 1 + workers: 2 + gpus: 1 + env: + PYTHONUNBUFFERED: "1" + args: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + gpu-memory-utilization: 0.5 + max-model-len: 4096 + +benchmark: + type: "sa-bench" + isl: 128 + osl: 128 + concurrencies: "4x8" diff --git a/examples/vllm/llm-d-disagg.yaml b/examples/vllm/llm-d-disagg.yaml new file mode 100644 index 000000000..d3df5ba59 --- /dev/null +++ b/examples/vllm/llm-d-disagg.yaml @@ -0,0 +1,94 @@ +# vLLM disaggregated prefill/decode behind the llm-d router (docs/llm-d.md). +# +# One TP1 prefill and one TP1 decode `vllm serve` worker sharing a node, KV over +# NIXL. On the router node srtctl starts llm-d's Endpoint Picker (EPP), which +# reads the workers from an endpoints file srtctl writes, and Envoy on the public +# port. The decode worker sits behind llm-d's P/D sidecar, a service the +# frontend implies (`llm-d-sidecar`, one per decode worker, started before the +# workers): the EPP picks the decode sidecar and a prefill worker for each +# request, and the sidecar runs the prefill before handing the KV transfer +# parameters to its vLLM. +# +# Aliases (`qwen3-0.6b`, `llm-d-vllm`) resolve through srtslurm.yaml; see +# examples/README.md. `llm-d-vllm` is a vLLM image that also ships the `epp`, +# `pd-sidecar`, and `envoy` executables on PATH (docs/llm-d.md#images). + +schema: 2 +name: "qwen3-0.6b-vllm-llm-d-disagg" + +slurm: + time_limit: "00:30:00" + +model: + path: "qwen3-0.6b" + container: "llm-d-vllm" + precision: "bf16" + +resources: + gpu_type: "h100" + gpus_per_node: 8 +frontend: + type: llm-d + enable_multiple_frontends: false + # The EPP's scheduler: decode and prefill profiles, picked by the disaggregation + # profile handler; srtctl adds the file-discovery plugin. + epp_config: + plugins: + # Hashes each prompt's blocks into the prefix index the prefix-cache scorer reads. + - type: approx-prefix-cache-producer + - type: prefill-filter + - type: decode-filter + - type: prefix-cache-scorer + - type: queue-scorer + - type: always-disagg-pd-decider + - type: disagg-profile-handler + parameters: + deciders: + prefill: always-disagg-pd-decider + - type: max-score-picker + schedulingProfiles: + - name: prefill + plugins: + - pluginRef: prefill-filter + - pluginRef: prefix-cache-scorer + weight: 2 + - pluginRef: queue-scorer + - pluginRef: max-score-picker + - name: decode + plugins: + - pluginRef: decode-filter + - pluginRef: queue-scorer + - pluginRef: max-score-picker + +engine: + type: vllm + connector: nixl +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 1 + env: + PYTHONUNBUFFERED: "1" + args: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + gpu-memory-utilization: 0.5 + max-model-len: 4096 + decode: + nodes: colocate + workers: 1 + gpus: 1 + env: + PYTHONUNBUFFERED: "1" + args: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + gpu-memory-utilization: 0.5 + max-model-len: 4096 + +benchmark: + type: "sa-bench" + isl: 128 + osl: 128 + concurrencies: "4x8" diff --git a/mkdocs.yml b/mkdocs.yml index 06020cf42..61776d3aa 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -74,6 +74,7 @@ nav: - SGLang Router: sglang-router.md - vLLM Router: vllm-router.md - Shepherd Model Gateway: smg.md + - llm-d Router: llm-d.md - Benchmarks: - Benchmarks: benchmarks.md - Accuracy Benchmarks: accuracy.md diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index ecf0be3cb..f0c959379 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -437,6 +437,32 @@ def kv_connector_for_mode(self, mode: WorkerMode) -> KVConnector | None: """The connector table row for a mode; None for no connector or a raw JSON ``--kv-transfer-config``.""" return kv_connector_row(self.connector_for_mode(mode)) + def kv_connector_class(self, mode: WorkerMode) -> str | None: + """The vLLM connector class a mode's workers run: ``kv_connector`` of their ``--kv-transfer-config``. + + A ``kv-transfer-config`` written in the role's args wins, as it does on the + command line; otherwise the mode's connector (a table row or raw JSON). + None when the mode runs no connector. + """ + explicit = next( + ( + value + for key, value in self.get_config_for_mode(mode).items() + if normalize_vllm_config_key(key) == "kv-transfer-config" + ), + None, + ) + if explicit is not None: + payload = explicit if isinstance(explicit, Mapping) else json.loads(str(explicit)) + return payload.get("kv_connector") + row = self.kv_connector_for_mode(mode) + if row is not None: + return row.kv_connector + connector = self.connector_for_mode(mode) + if not connector or connector.lower() in ("null", "none"): + return None + return json.loads(connector).get("kv_connector") + def discovers_workers(self) -> bool: """Whether the prefill/decode workers register with the router over its discovery endpoint. diff --git a/src/srtctl/cli/mixins/service_stage.py b/src/srtctl/cli/mixins/service_stage.py index 129d485d0..5bcf74d2d 100644 --- a/src/srtctl/cli/mixins/service_stage.py +++ b/src/srtctl/cli/mixins/service_stage.py @@ -143,7 +143,8 @@ def service_nodes(self, service: ServiceConfig) -> list[str]: def service_instances(self, service: ServiceConfig) -> list[tuple[str, Process | None]]: """The instances a service launches: ``(node, None)`` per placed node, or under - ``placement.per: worker`` ``(node, process)`` per engine worker on those nodes. + ``placement.per: worker`` ``(node, process)`` per engine worker on those nodes + that the kind attaches to (``ServiceKind.attaches_to``). A worker's instance is attached to its engine 0 process (a worker with shadow engines has several processes on a node; the sidecar serves them all). @@ -155,12 +156,14 @@ def service_instances(self, service: ServiceConfig) -> list[tuple[str, Process | return [(node, None) for node in nodes] where = service.effective_placement order = {node: i for i, node in enumerate(nodes)} + kind = get_service_kind(service.type) attached = [ process for process in self.backend_processes if process.node in order and getattr(process, "engine_id", 0) == 0 and (where == "workers" or process.endpoint_mode == where) + and kind.attaches_to(process) ] attached.sort(key=lambda p: (order[p.node], p.endpoint_index, p.node_rank)) return [(process.node, process) for process in attached] diff --git a/src/srtctl/core/AGENTS.md b/src/srtctl/core/AGENTS.md index 1652efe92..5be5a16c5 100644 --- a/src/srtctl/core/AGENTS.md +++ b/src/srtctl/core/AGENTS.md @@ -4,7 +4,7 @@ Rules for `src/srtctl/core/` (schema, runtime, topology, processes, health). Sta ## Ports -Fixed ports are constants in `src/srtctl/ports.py`. Every port a worker process binds is a `PortKind` in the same module (`name`, `base`, `stride`, `per_node`) and is handed out by `NodePortAllocator.next(kind, node, size)` in `src/srtctl/core/topology.py`: per-node counters for listeners only bound on that node (`HTTP_PORTS`, `BOOTSTRAP_PORTS`, `DP_RPC_PORTS`, `DIST_INIT_PORTS`, `TOKENSPEED_PORTS`) and global counters for side channels peers address (`SYS_PORTS`, `KV_EVENTS_PORTS`, `NIXL_PORTS`, `KVBM_ZMQ_PORTS`, `NCCL_PORTS`, `VLLM_SCAN_PORTS`, `MORIIO_HANDSHAKE_PORTS`, `MORIIO_NOTIFY_PORTS`, `TRTLLM_DIST_INIT_PORTS`, `SIDECAR_GRPC_PORTS`). `size > 1` reserves a block when the engine adds a rank offset to the port it is given (MoRI-IO adds the local DP and TP rank to its notify base). `endpoints_to_processes` allocates the generic kinds and stores them on `Process`; a backend adds its engine-specific kinds with `dataclasses.replace` on the way out (SGLang `nccl_port` and `dist_init_port`, vLLM `vllm_scan_port` or, for a discovery connector, `moriio_handshake_port` and `moriio_notify_port`, TRT-LLM `trtllm_dist_init_port`, TokenSpeed `tokenspeed_port` and a `dist_init_port` block). `do_sweep` builds the one allocator per job, seeding the sidecar gRPC base from `dynamo.sidecar_port`. +Fixed ports are constants in `src/srtctl/ports.py`. Every port a worker process binds is a `PortKind` in the same module (`name`, `base`, `stride`, `per_node`) and is handed out by `NodePortAllocator.next(kind, node, size)` in `src/srtctl/core/topology.py`: per-node counters for listeners only bound on that node (`HTTP_PORTS`, `BOOTSTRAP_PORTS`, `DP_RPC_PORTS`, `DIST_INIT_PORTS`, `TOKENSPEED_PORTS`) and global counters for side channels peers address (`SYS_PORTS`, `KV_EVENTS_PORTS`, `NIXL_PORTS`, `KVBM_ZMQ_PORTS`, `NCCL_PORTS`, `VLLM_SCAN_PORTS`, `MORIIO_HANDSHAKE_PORTS`, `MORIIO_NOTIFY_PORTS`, `TRTLLM_DIST_INIT_PORTS`, `SIDECAR_GRPC_PORTS`); `PROXY_PORTS` is per node and is allocated after the backends by `worker_processes` (`core/worker_backends.py`), for the routable workers of the modes the frontend proxies (`Frontend.proxied_worker_modes`). `size > 1` reserves a block when the engine adds a rank offset to the port it is given (MoRI-IO adds the local DP and TP rank to its notify base). `endpoints_to_processes` allocates the generic kinds and stores them on `Process`; a backend adds its engine-specific kinds with `dataclasses.replace` on the way out (SGLang `nccl_port` and `dist_init_port`, vLLM `vllm_scan_port` or, for a discovery connector, `moriio_handshake_port` and `moriio_notify_port`, TRT-LLM `trtllm_dist_init_port`, TokenSpeed `tokenspeed_port` and a `dist_init_port` block). `do_sweep` builds the one allocator per job, seeding the sidecar gRPC base from `dynamo.sidecar_port`. Rules that follow: a new listener is a new `PortKind` plus a `Process` field, allocated in the topology builder, never `some_base + (sys_port - DYN_SYSTEM_PORT_BASE)` or any other arithmetic on another port at command-build time. Consumers read the field; a `None` means the topology was built without that kind, and a consumer that needs it raises rather than guessing. A worker's own address is `get_hostname_ip(node, runtime.network_interface)`, not upstream's interface guess. Fixed scan bases such as `VLLM_PORT` exist only to keep co-located `get_open_port()` scans apart; if a connector allocates inside forked children, leave the base unset so the kernel assigns ports. Frontends bind `FRONTEND_PUBLIC_PORT` (8000), or `FRONTEND_INTERNAL_PORT` (8180) behind nginx when `enable_multiple_frontends` is set. Service ports are `ServiceKind` defaults overridden by `options`. `tests/test_port_allocator.py` asserts over every example recipe that no two processes on a node share a port and global kinds never repeat. diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 24d4987a4..abc31c7fa 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -2530,7 +2530,8 @@ class FrontendConfig: Attributes: type: Frontend type - "dynamo" (default); "sglang-router" (SGLang Model - Gateway), "vllm-router", "smg" (Shepherd Model Gateway, any backend), "atomesh", and + Gateway), "vllm-router", "smg" (Shepherd Model Gateway, any backend), "llm-d" + (llm-d Endpoint Picker behind Envoy, vLLM), "atomesh", and "tilert-router" (static routers); "sglang", "vllm", and "trtllm_serve" (direct: the single aggregate worker binds the public port, no router process); "none" (services-only job: no router, no @@ -2564,6 +2565,13 @@ class FrontendConfig: writes this mapping under the top-level ``worker_selection`` key in a generated router policy YAML and passes it to the Dynamo frontend via ``--router-policy-config``. + epp_config: llm-d Endpoint Picker configuration (``EndpointPickerConfig``: + ``plugins``, ``schedulingProfiles``, ...) for ``frontend.type: llm-d``. + srtctl writes it to a file with the ``file-discovery`` plugin and + ``dataLayer.discovery`` added, so the EPP reads the job's workers from + the endpoints file srtctl renders. Required for prefill/decode jobs + (the scheduler's P/D profiles); omitted, an aggregate job runs the + EPP's built-in default profile. args: CLI arguments passed to the frontend/router process env: Environment variables for frontend processes container_image: Optional router-specific image. Static routers use the @@ -2583,6 +2591,7 @@ class FrontendConfig: nginx_session_affinity_header: str = "X-Dynamo-Session-ID" nginx_keepalive_timeout: str = "600s" worker_selection: dict[str, Any] | None = None + epp_config: dict[str, Any] | None = None args: dict[str, Any] | None = None env: dict[str, str] | None = None container_image: str | None = None diff --git a/src/srtctl/core/topology.py b/src/srtctl/core/topology.py index 2ca30a526..eb05962e7 100644 --- a/src/srtctl/core/topology.py +++ b/src/srtctl/core/topology.py @@ -162,6 +162,8 @@ class Process: their own ports and their own srun step. kvbm_zmq_port: KVBM leader ZMQ pub port (ack is the next port); the leader's is used sidecar_grpc_port: Dynamo sidecar gRPC listener, allocated when the job runs sidecars + proxy_port: listener of the proxy the router sends this routable worker's traffic + through (llm-d's P/D sidecar), allocated when the frontend proxies its mode nccl_port: SGLang local TP rendezvous port, one per server process dist_init_port: SGLang multi-node dist-init port; the same value on every process of an endpoint grpc_http_port: HTTP sidecar (/metrics, profiler routes) of an SGLang gRPC-mode leader @@ -191,6 +193,7 @@ class Process: engine_id: int = 0 kvbm_zmq_port: int | None = None sidecar_grpc_port: int | None = None + proxy_port: int | None = None nccl_port: int | None = None dist_init_port: int | None = None grpc_http_port: int | None = None diff --git a/src/srtctl/core/worker_backends.py b/src/srtctl/core/worker_backends.py index 0ad8bf753..7e6db0683 100644 --- a/src/srtctl/core/worker_backends.py +++ b/src/srtctl/core/worker_backends.py @@ -11,11 +11,12 @@ from __future__ import annotations from collections.abc import Sequence +from dataclasses import replace from functools import partial from typing import TYPE_CHECKING from srtctl.core.topology import Endpoint, NodePortAllocator, Process, allocate_endpoints -from srtctl.ports import SIDECAR_GRPC_PORTS +from srtctl.ports import PROXY_PORTS, SIDECAR_GRPC_PORTS if TYPE_CHECKING: from srtctl.backends import BackendConfig @@ -65,10 +66,26 @@ def expand(backend: BackendConfig, selected: list[Endpoint]) -> list[Process]: ) if not config.has_role_backends: - return expand(config.backend, endpoints) - processes: list[Process] = [] - for role in ("prefill", "decode", "agg"): - selected = [endpoint for endpoint in endpoints if endpoint.mode == role] - if selected: - processes.extend(expand(config.backend_for_role(role), selected)) - return processes + processes = expand(config.backend, endpoints) + else: + processes = [] + for role in ("prefill", "decode", "agg"): + selected = [endpoint for endpoint in endpoints if endpoint.mode == role] + if selected: + processes.extend(expand(config.backend_for_role(role), selected)) + return with_proxy_ports(config, processes, allocator) + + +def with_proxy_ports(config: SrtConfig, processes: list[Process], allocator: NodePortAllocator) -> list[Process]: + """Give every routable worker of a mode the frontend proxies the listener its proxy binds.""" + from srtctl.frontends import FRONTEND_NONE, get_frontend + + if config.frontend.type == FRONTEND_NONE: + return processes + proxied = get_frontend(config.frontend.type).proxied_worker_modes(config) + return [ + replace(process, proxy_port=allocator.next(PROXY_PORTS, process.node)) + if process.http_port > 0 and process.endpoint_mode in proxied + else process + for process in processes + ] diff --git a/src/srtctl/frontends/AGENTS.md b/src/srtctl/frontends/AGENTS.md index c648ad7f5..bc4f1e39b 100644 --- a/src/srtctl/frontends/AGENTS.md +++ b/src/srtctl/frontends/AGENTS.md @@ -4,7 +4,7 @@ Rules for `src/srtctl/frontends/`. The Design Rules in the root `CLAUDE.md` appl ## Frontends and readiness -The frontend is the process that owns the public OpenAI port (`FRONTEND_PUBLIC_PORT`) and decides when the job is ready. Implementations live under `src/srtctl/frontends/`, register with `@register_frontend("")`, and are imported from the package `__init__`; `frontend.type` resolves through that registry (`get_frontend`, `list_frontend_types`) and nowhere else. Registered today: `dynamo` (etcd/NATS discovery), `sglang-router`, `vllm-router` and `smg` (static URL routers built on `StaticRouterFrontend`; `smg` pairs with any backend), `trtllm_serve` (direct aggregate or the disaggregated orchestrator), `sglang` and `vllm` (one direct worker owns the port); `none` is the services-only job with no implementation and no gate. +The frontend is the process that owns the public OpenAI port (`FRONTEND_PUBLIC_PORT`) and decides when the job is ready. Implementations live under `src/srtctl/frontends/`, register with `@register_frontend("")`, and are imported from the package `__init__`; `frontend.type` resolves through that registry (`get_frontend`, `list_frontend_types`) and nowhere else. Registered today: `dynamo` (etcd/NATS discovery), `sglang-router`, `vllm-router` and `smg` (static URL routers built on `StaticRouterFrontend`; `smg` pairs with any backend), `llm-d` (the llm-d EPP behind Envoy, on `StaticRouterFrontend`: it writes the workers into an endpoints file instead of a command line, and reaches decode workers through the P/D sidecar service it implies), `trtllm_serve` (direct aggregate or the disaggregated orchestrator), `sglang` and `vllm` (one direct worker owns the port); `none` is the services-only job with no implementation and no gate. `Frontend` hooks: `required_backend` and `validate(config)` (recipe rules, run by `SrtConfig._validate_frontend` at load so dry-run catches them; raise `ValueError` with the user-facing message), `health_endpoint` and `parse_health` (readiness), `get_backend_health_urls` (second gate: every advertised worker URL must answer 200 before traffic), `start_frontends` (launch on `topology.frontend_nodes`, one `ManagedProcess` per node with a `step_name`), `get_frontend_args_list` (`frontend.args` to CLI). The schema knows no individual frontend: pairing and per-type rules come from these two members. @@ -14,6 +14,8 @@ Readiness runs in `BenchmarkStageMixin._wait_for_service_ready`: `wait_for_model The frontend also owns the worker shape, and backends read it instead of comparing names: `worker_launch` (`dynamo` workers register with the Dynamo runtime, `direct` workers are the engine's own server), `worker_api_port(mode)` (`public` when the worker is itself the endpoint and binds `runtime.frontend_port`, `allocated` when a router fronts it on `Process.http_port`), and `expands_node_local_dp` (vLLM Router expands per-node hybrid-LB pools, so the vLLM backend launches one API per node and refuses `per_gpu`). `build_worker_command` still takes `frontend_type` and resolves it through `get_frontend`; a new router mode is therefore one frontend override plus, at most, a new attribute on the protocol. +A frontend that reaches a mode's workers through a per-worker proxy names the modes in `proxied_worker_modes(config)`: `worker_processes` (`core/worker_backends.py`) then allocates `Process.proxy_port` (`PROXY_PORTS`) for every routable worker of those modes, and the service that binds it attaches only to those processes (`ServiceKind.attaches_to`). llm-d's P/D sidecar (`services/llm_d_sidecar.py`) is the user. + Which rank serves what is the frontend's call too, one method per consumer: `worker_metrics_port(process, runtime)` and `worker_metrics_path(backend, mode)` feed the tachometer scrape targets and the AIPerf metrics URLs (a direct worker answers at `backend.prometheus_metrics_path`; `metrics_path` is the router's own) (Dynamo: every rank on its system port; native servers: the leader or each routable pool on its HTTP port), `worker_endpoint_port(process, config, runtime)` feeds the `PREFILL_IPS`-style benchmark env (one per logical worker), `profiling_control_port` and `profiling_control_is_leader_only` feed iteration-triggered nsys control, `direct_endpoint_nodes(processes)` names the nodes whose worker is itself the public endpoint, and `worker_ready_port(process)` is what sequential endpoint start polls. `core/telemetry.py` and the benchmark stage contain no frontend-name checks; `BenchmarkStageMixin.frontend` is `None` for a services-only job. Expected counts come from the frontend too: `health_expectations(config, processes)` returns `(prefill, decode, description)` in the units its endpoint reports. Aggregate workers count as decode; Dynamo counts vLLM DP registrations, vLLM Router counts the ranks it expands each URL into, everything else counts logical workers. diff --git a/src/srtctl/frontends/__init__.py b/src/srtctl/frontends/__init__.py index 4c9f6c9ff..611e15b43 100644 --- a/src/srtctl/frontends/__init__.py +++ b/src/srtctl/frontends/__init__.py @@ -10,6 +10,7 @@ Supported frontend types: - dynamo: Dynamo frontend with NATS/etcd communication +- llm-d: llm-d Endpoint Picker behind Envoy, with the P/D sidecar on decode workers - sglang: Direct sglang.launch_server for a single aggregate worker (no router) - sglang-router: SGLang Model Gateway router in front of static workers - smg: Shepherd Model Gateway in front of any engine's static workers @@ -29,6 +30,7 @@ ) from srtctl.frontends.dynamic_frontend import DynamicFrontend from srtctl.frontends.dynamo import DynamoFrontend +from srtctl.frontends.llm_d import LLMDFrontend from srtctl.frontends.sglang import SGLangRouterFrontend from srtctl.frontends.sglang_direct import SGLangFrontend from srtctl.frontends.smg import SMGFrontend @@ -43,6 +45,7 @@ "DynamicFrontend", "DynamoFrontend", "Frontend", + "LLMDFrontend", "SGLangFrontend", "SGLangRouterFrontend", "SMGFrontend", diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py index 6702f1a63..94fdd3245 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py @@ -180,6 +180,15 @@ def implied_services(self, config: Any) -> list["EffectiveService"]: """Services this frontend needs that the recipe did not name (Dynamo: its discovery plane).""" return [] + def proxied_worker_modes(self, config: Any) -> frozenset[str]: + """Worker modes the router reaches through a per-worker proxy instead of the worker's own port. + + Every routable worker of these modes gets ``Process.proxy_port`` from the + allocator (``core.worker_backends.worker_processes``); the frontend implies + the service that binds it (llm-d: its P/D sidecar on decode workers). + """ + return frozenset() + def frontend_metrics_port(self, frontend_args: dict[str, Any] | None) -> int | None: """Port of a Prometheus listener separate from the routing port, or ``None`` when metrics share it.""" return None diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py new file mode 100644 index 000000000..3af2af24e --- /dev/null +++ b/src/srtctl/frontends/llm_d.py @@ -0,0 +1,335 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""llm-d router frontend (`frontend.type: llm-d`): the Endpoint Picker behind Envoy. + +llm-d routes with two processes on the router node. Envoy owns the public port +and asks the Endpoint Picker (EPP) over ext_proc which worker each request goes +to; the EPP answers with the ``x-gateway-destination-endpoint`` header and +Envoy's ORIGINAL_DST cluster forwards there. For prefill/decode the EPP picks a +decode endpoint and names the prefill worker in ``x-prefiller-host-port``; that +decode endpoint is llm-d's P/D sidecar, a service this frontend implies on every +routable decode worker (``services/llm_d_sidecar.py``). + +Without Kubernetes the EPP reads its workers from a file (the ``file-discovery`` +plugin), so srtctl is the registrar: once every worker answers ``/health`` it +writes the endpoints file, the EPP configuration (the recipe's +``frontend.epp_config`` plus the discovery plugin), and the Envoy configuration +into the log directory, then starts the EPP and Envoy. The job is ready when +Envoy's admin ``/ready`` answers and the EPP reports every endpoint ready +(``llm_d_epp_ready_endpoints``: endpoints whose metrics it scrapes). + +Upstream, pinned: llm-d-router v0.11.0 +(https://github.com/llm-d/llm-d-router/tree/a5cbe600ebade00cf3e9885beaf2bfacddeabce1): +``cmd/epp/runner/runner.go`` (``runWithFileDiscovery``), +``pkg/epp/framework/plugins/datalayer/discovery/file``; the Envoy configuration +follows llm-d's no-Kubernetes guide +(https://github.com/llm-d/llm-d/blob/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment/router/envoy/envoy.yaml). +""" + +from __future__ import annotations + +import copy +import logging +import shlex +import threading +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar + +import requests +import yaml + +from srtctl.core.health import WorkerHealthResult +from srtctl.frontends.base import numactl_prefix, register_frontend +from srtctl.frontends.static_router import StaticRouterFrontend +from srtctl.ports import ( + LLM_D_ENVOY_ADMIN_PORT, + LLM_D_EPP_GRPC_PORT, + LLM_D_EPP_HEALTH_PORT, + LLM_D_EPP_METRICS_PORT, +) +from srtctl.services.config import ServiceConfig, ServicePlacementConfig +from srtctl.services.implicit import EffectiveService +from srtctl.services.llm_d_sidecar import LLM_D_SIDECAR_TYPE, sidecar_kv_connector + +if TYPE_CHECKING: + from srtctl.core.processes import ManagedProcess + from srtctl.core.runtime import RuntimeContext + from srtctl.core.topology import Process + +logger = logging.getLogger(__name__) + +# Pool the EPP serves; also the namespace of every endpoint in the endpoints file. +POOL = "srtctl" +DISCOVERY_PLUGIN = "srtctl-file-discovery" +ENDPOINTS_FILE = "llm-d-endpoints.yaml" +EPP_CONFIG_FILE = "llm-d-epp-config.yaml" +ENVOY_CONFIG_FILE = "llm-d-envoy.yaml" +ENVOY_ACCESS_LOG = "llm-d-envoy-access.log" +# The EPP's role label (pkg/epp/framework/plugins/scheduling/filter/bylabel/roles.go): +# prefill-filter and decode-filter select on it; an aggregate worker serves both. +ROLE_LABEL = "llm-d.ai/role" +ROLE_BY_MODE = {"prefill": "prefill", "decode": "decode", "agg": "both"} +# Gauge of endpoints whose metrics the EPP scraped within its staleness window +# (pkg/epp/metrics/llm_d_router_metrics.go). +READY_ENDPOINTS_METRIC = "llm_d_epp_ready_endpoints" +# EPP flags srtctl sets; frontend.args may not repeat them. +_MANAGED_EPP_FLAGS = frozenset( + {"pool-name", "pool-namespace", "config-file", "config-text", "grpc-port", "grpc-health-port", "metrics-port"} +) + + +def _is_pd(config: Any) -> bool: + topology = config.topology + return topology.num_prefill > 0 and topology.num_decode > 0 + + +def _container_path(name: str) -> str: + """``log_dir/`` as every container sees it (``log_dir`` is mounted at ``/logs``).""" + return str(Path("/logs") / name) + + +def epp_config_document(epp_config: dict[str, Any] | None, endpoints_path: str) -> dict[str, Any]: + """The recipe's ``EndpointPickerConfig`` with srtctl's file discovery added. + + ``dataLayer.discovery.pluginRef`` is the spelling both v0.10 and v0.11 accept + (v0.11 also reads ``discovery.endpoints.pluginRef``). + """ + document: dict[str, Any] = {"apiVersion": "llm-d.ai/v1alpha1", "kind": "EndpointPickerConfig"} + document.update(copy.deepcopy(epp_config or {})) + document["plugins"] = [ + *(document.get("plugins") or []), + { + "name": DISCOVERY_PLUGIN, + "type": "file-discovery", + # srtctl writes the file once, from the workers it launched. + "parameters": {"path": endpoints_path, "watchFile": False}, + }, + ] + document["dataLayer"] = {**(document.get("dataLayer") or {}), "discovery": {"pluginRef": DISCOVERY_PLUGIN}} + return document + + +def envoy_config(listen_port: int, access_log: str) -> str: + """Envoy in front of the EPP: ext_proc to it on localhost, then ORIGINAL_DST to the endpoint it picked.""" + from jinja2 import Environment, FileSystemLoader + + templates = Environment(loader=FileSystemLoader(str(Path(__file__).parent.parent / "templates"))) + return templates.get_template("llm-d-envoy.yaml.j2").render( + admin_port=LLM_D_ENVOY_ADMIN_PORT, + listen_port=listen_port, + access_log=access_log, + epp_port=LLM_D_EPP_GRPC_PORT, + ) + + +def parse_ready_endpoints(metrics_text: str) -> int | None: + """The EPP's ready-endpoint gauge from its Prometheus text, or None before it is published.""" + total: float | None = None + for line in metrics_text.splitlines(): + if line.startswith((f"{READY_ENDPOINTS_METRIC} ", f"{READY_ENDPOINTS_METRIC}{{")): + total = (total or 0.0) + float(line.rsplit(" ", 1)[1]) + return None if total is None else int(total) + + +@register_frontend("llm-d") +class LLMDFrontend(StaticRouterFrontend): + """llm-d Endpoint Picker behind Envoy, in front of direct vLLM workers.""" + + type: ClassVar[str] = "llm-d" + required_backend: ClassVar[str | None] = "vllm" + executable: ClassVar[tuple[str, ...]] = ("epp",) + # The EPP takes prefill/decode from its scheduler configuration, not a flag. + pd_flag: ClassVar[str] = "" + process_name: ClassVar[str] = "llm-d" + # The endpoints file is written once, from workers that already answer /health. + wait_for_workers_before_start: ClassVar[bool] = True + + def validate(self, config: Any) -> None: + frontend = config.frontend + if ( + frontend.enable_multiple_frontends + and frontend.num_additional_frontends > 0 + and config.engine_node_count > 1 + ): + raise ValueError( + "frontend.type: llm-d runs one Endpoint Picker, which keeps its routing state in memory; " + "set frontend.enable_multiple_frontends: false (or num_additional_frontends: 0)" + ) + topology = config.topology + for mode, count in ( + ("prefill", topology.num_prefill), + ("decode", topology.num_decode), + ("agg", topology.num_agg), + ): + if count and config.backend_for_role(mode).is_grpc_mode(mode): + raise ValueError(f"frontend.type: llm-d routes HTTP workers; roles.{mode} serves gRPC") + epp_config = frontend.epp_config or {} + data_layer = epp_config.get("dataLayer") or {} + if "discovery" in data_layer or any( + plugin.get("type") == "file-discovery" for plugin in epp_config.get("plugins") or [] + ): + raise ValueError( + "frontend.epp_config must not configure endpoint discovery: srtctl adds the file-discovery " + "plugin and dataLayer.discovery for the workers it launches" + ) + if _is_pd(config): + if not epp_config.get("schedulingProfiles"): + raise ValueError( + "frontend.type: llm-d with prefill and decode workers needs frontend.epp_config with " + "prefill and decode schedulingProfiles and a disaggregation profile handler" + ) + sidecar_kv_connector(config.backend_for_role("decode")) + + def proxied_worker_modes(self, config: Any) -> frozenset[str]: + """Prefill/decode: the router reaches each decode worker through its P/D sidecar.""" + return frozenset({"decode"}) if _is_pd(config) else frozenset() + + def implied_services(self, config: Any) -> list[EffectiveService]: + if not _is_pd(config): + return [] + return [ + EffectiveService( + ServiceConfig( + name=LLM_D_SIDECAR_TYPE, + type=LLM_D_SIDECAR_TYPE, + placement=ServicePlacementConfig(node="decode", per="worker"), + # Nothing on the sidecar's path registers with etcd or NATS. + inherit_discovery_env=False, + ), + implicit=True, + reason="frontend.type llm-d (prefill/decode)", + ) + ] + + def frontend_metrics_port(self, frontend_args: dict[str, Any] | None) -> int | None: + """The EPP serves Prometheus on its own listener.""" + return LLM_D_EPP_METRICS_PORT + + def health_expectations(self, config: Any, processes: list[Process] | None) -> tuple[int, int, str]: + """One endpoint per routable worker process: its prefill vLLM, its decode sidecar, or its aggregate vLLM.""" + if processes is None: + return super().health_expectations(config, processes) + routable = [process for process in processes if process.http_port > 0] + prefill = sum(process.endpoint_mode == "prefill" for process in routable) + decode = len(routable) - prefill + return prefill, decode, f"{len(routable)} llm-d endpoints" + + def probe_ready( + self, host: str, port: int, expected_prefill: int, expected_decode: int, config: Any + ) -> WorkerHealthResult: + """Envoy's admin ``/ready``, then the EPP's count of endpoints whose metrics it scrapes.""" + envoy = requests.get(f"http://{host}:{LLM_D_ENVOY_ADMIN_PORT}/ready", timeout=5.0) + if envoy.status_code != 200: + return WorkerHealthResult(ready=False, message=f"Envoy /ready returned HTTP {envoy.status_code}") + metrics = requests.get(f"http://{host}:{LLM_D_EPP_METRICS_PORT}/metrics", timeout=5.0) + expected = expected_prefill + expected_decode + ready = parse_ready_endpoints(metrics.text) if metrics.status_code == 200 else None + if ready is None: + return WorkerHealthResult(ready=False, message=f"EPP has not published {READY_ENDPOINTS_METRIC} yet") + return WorkerHealthResult( + ready=ready >= expected, + message=f"llm-d EPP reports {ready}/{expected} endpoints ready", + decode_ready=ready, + decode_expected=expected, + ) + + def endpoints_document( + self, backend_processes: list[Process], network_interface: str | None + ) -> dict[str, list[dict[str, Any]]]: + """The file-discovery endpoints: every routable worker, at its proxy's port when it has one.""" + endpoints = [] + for process in backend_processes: + if process.http_port <= 0: + continue + port = process.proxy_port if process.proxy_port is not None else process.http_port + endpoints.append( + { + "name": f"{process.endpoint_mode}-{process.endpoint_index}-{process.node_rank}", + "namespace": POOL, + "address": self.resolve_worker_host(process.node, network_interface), + "port": str(port), + "labels": {ROLE_LABEL: ROLE_BY_MODE[process.endpoint_mode]}, + } + ) + return {"endpoints": endpoints} + + def epp_command(self, config: Any, epp_config_path: str) -> list[str]: + user_args = self.get_frontend_args_list(config.frontend.args) + managed = {str(key).replace("_", "-") for key in (config.frontend.args or {})} & _MANAGED_EPP_FLAGS + if managed: + raise ValueError(f"frontend.args sets {', '.join(sorted(managed))}, which srtctl manages for llm-d") + return [ + *self.executable, + f"--pool-name={POOL}", + f"--pool-namespace={POOL}", + f"--config-file={epp_config_path}", + f"--grpc-port={LLM_D_EPP_GRPC_PORT}", + f"--grpc-health-port={LLM_D_EPP_HEALTH_PORT}", + f"--metrics-port={LLM_D_EPP_METRICS_PORT}", + # Envoy dials the EPP over plaintext HTTP/2 on localhost. + "--secure-serving=false", + *user_args, + ] + + def start_frontends( + self, + topology: Any, + runtime: RuntimeContext, + config: Any, + backend: Any, + backend_processes: list[Process], + stop_event: threading.Event | None = None, + ) -> list[ManagedProcess]: + from srtctl.core.processes import FRONTEND_TERMINATE_TIMEOUT_SECONDS, ManagedProcess + + self.wait_for_workers( + self.collect_workers(backend, backend_processes, runtime.network_interface), config, stop_event + ) + + endpoints = self.endpoints_document(backend_processes, runtime.network_interface) + (runtime.log_dir / ENDPOINTS_FILE).write_text(yaml.safe_dump(endpoints, sort_keys=False)) + epp_document = epp_config_document(config.frontend.epp_config, _container_path(ENDPOINTS_FILE)) + (runtime.log_dir / EPP_CONFIG_FILE).write_text(yaml.safe_dump(epp_document, sort_keys=False)) + (runtime.log_dir / ENVOY_CONFIG_FILE).write_text( + envoy_config(topology.frontend_port, _container_path(ENVOY_ACCESS_LOG)) + ) + logger.info("llm-d endpoints (%d): %s", len(endpoints["endpoints"]), endpoints["endpoints"]) + + commands = { + "epp": self.epp_command(config, _container_path(EPP_CONFIG_FILE)), + # One Envoy per step: no hot-restart sockets shared with another Envoy on the host network. + "envoy": ["envoy", "-c", _container_path(ENVOY_CONFIG_FILE), "--disable-hot-restart"], + } + container_image = config.frontend.container_image or str(runtime.container_image) + env = {**runtime.environment, **(config.frontend.env or {})} + processes: list[ManagedProcess] = [] + for idx, node in enumerate(topology.frontend_nodes): + for component, command in commands.items(): + cmd = numactl_prefix(config) + command + step_name = f"{self.process_name}-{component}_{idx}" + log_file = runtime.log_dir / f"{node}_{self.process_name}-{component}_{idx}.out" + logger.info("Starting llm-d %s %d on %s: %s", component, idx, node, shlex.join(cmd)) + proc = self.start_process( + command=cmd, + nodelist=[node], + output=str(log_file), + container_image=container_image, + container_mounts=runtime.container_mounts, + env_to_set=env or None, + het_group=runtime.nodes.het_group_for(node), + step_name=step_name, + srun_options=runtime.srun_options, + ) + processes.append( + ManagedProcess( + name=step_name, + popen=proc, + log_file=log_file, + node=node, + critical=True, + terminate_timeout=FRONTEND_TERMINATE_TIMEOUT_SECONDS, + step_name=step_name, + ) + ) + return processes diff --git a/src/srtctl/frontends/static_router.py b/src/srtctl/frontends/static_router.py index c727796db..34ff03ddc 100644 --- a/src/srtctl/frontends/static_router.py +++ b/src/srtctl/frontends/static_router.py @@ -224,6 +224,43 @@ def get_backend_health_urls( """Return extra direct readiness requirements, if any.""" return [] + def wait_for_workers( + self, workers: list[RouterWorker], config: Any, stop_event: threading.Event | None = None + ) -> None: + """Block until every advertised worker is healthy, within the recipe's health-check budget. + + HTTP workers must answer ``/health`` with 200; a gRPC worker serves no HTTP + ``/health``, so its port accepting connections is the signal. Raises when + the budget runs out or ``stop_event`` is set. + """ + if config.health_check is None: + return + health_check = config.health_check + timeout = float(health_check.max_attempts * health_check.interval_seconds) + health_urls = [ + f"{worker.url.rstrip('/')}/health" for worker in workers if worker.url.startswith(("http://", "https://")) + ] + grpc_targets = [urlsplit(worker.url) for worker in workers if worker.url.startswith("grpc://")] + logger.info("Waiting for %d advertised backend endpoints before starting %s", len(workers), self.type) + deadline = time.monotonic() + timeout + ready = all( + wait_for_port( + str(target.hostname), + int(target.port or 0), + timeout=max(deadline - time.monotonic(), 0.0), + stop_event=stop_event, + ) + for target in grpc_targets + ) and wait_for_http_endpoints( + health_urls, + poll_interval=float(health_check.interval_seconds), + timeout=max(deadline - time.monotonic(), 0.0), + report_every=60.0, + stop_event=stop_event, + ) + if not ready: + raise RuntimeError(f"Advertised backend endpoints did not become ready before {self.type} startup") + def discovers_workers(self, backend: Any) -> bool: """Whether the router learns its workers by registration instead of from its command line. @@ -283,35 +320,8 @@ def start_frontends( ) workers = self.collect_workers(backend, backend_processes, runtime.network_interface) - if self.wait_for_workers_before_start and config.health_check is not None: - health_check = config.health_check - timeout = float(health_check.max_attempts * health_check.interval_seconds) - health_urls = [ - f"{worker.url.rstrip('/')}/health" - for worker in workers - if worker.url.startswith(("http://", "https://")) - ] - # A gRPC worker serves no HTTP /health; its port accepting connections is the signal. - grpc_targets = [urlsplit(worker.url) for worker in workers if worker.url.startswith("grpc://")] - logger.info("Waiting for %d advertised backend endpoints before starting %s", len(workers), self.type) - deadline = time.monotonic() + timeout - ready = all( - wait_for_port( - str(target.hostname), - int(target.port or 0), - timeout=max(deadline - time.monotonic(), 0.0), - stop_event=stop_event, - ) - for target in grpc_targets - ) and wait_for_http_endpoints( - health_urls, - poll_interval=float(health_check.interval_seconds), - timeout=max(deadline - time.monotonic(), 0.0), - report_every=60.0, - stop_event=stop_event, - ) - if not ready: - raise RuntimeError(f"Advertised backend endpoints did not become ready before {self.type} startup") + if self.wait_for_workers_before_start: + self.wait_for_workers(workers, config, stop_event) processes: list[ManagedProcess] = [] for idx, node in enumerate(topology.frontend_nodes): diff --git a/src/srtctl/ports.py b/src/srtctl/ports.py index 6106f2d4e..46c1af916 100644 --- a/src/srtctl/ports.py +++ b/src/srtctl/ports.py @@ -42,6 +42,17 @@ # passed as --prometheus-port so srtctl, not upstream's default, owns it. SMG_METRICS_PORT = 29000 +# llm-d router (frontend.type: llm-d) on each router node: the Endpoint Picker's ext_proc gRPC, +# gRPC health, and Prometheus listeners (upstream's defaults, passed explicitly so srtctl owns +# them) and Envoy's admin listener, which serves /ready. +LLM_D_EPP_GRPC_PORT = 9002 +LLM_D_EPP_HEALTH_PORT = 9003 +LLM_D_EPP_METRICS_PORT = 9090 +LLM_D_ENVOY_ADMIN_PORT = 9901 +# First listener of a per-worker proxy a router sends a worker's traffic through +# (llm-d's P/D sidecar in front of a decode worker); see PROXY_PORTS. +WORKER_PROXY_PORT_BASE = 9600 + # TRT-LLM torch.distributed bootstrap, one port per MPI endpoint. TRTLLM_DIST_INIT_PORT_BASE = 29500 @@ -125,6 +136,9 @@ class PortKind: # KVBM leader ZMQ pair: pub at the port, ack at the port + 1. KVBM_ZMQ_PORTS = PortKind("kvbm_zmq", KVBM_ZMQ_PORT_BASE, 2) SIDECAR_GRPC_PORTS = PortKind("sidecar_grpc", DYNAMO_SIDECAR_GRPC_PORT) +# Bound by the proxy in front of a routable worker whose mode the frontend proxies +# (``Frontend.proxied_worker_modes``), on the worker's node. +PROXY_PORTS = PortKind("proxy", WORKER_PROXY_PORT_BASE, per_node=True) # Engine-specific: the backend allocates these for its own processes. NCCL_PORTS = PortKind("nccl", SGLANG_NCCL_PORT_BASE) DIST_INIT_PORTS = PortKind("dist_init", SGLANG_DIST_INIT_PORT_BASE, per_node=True) diff --git a/src/srtctl/services/__init__.py b/src/srtctl/services/__init__.py index 81e220712..5e9c82103 100644 --- a/src/srtctl/services/__init__.py +++ b/src/srtctl/services/__init__.py @@ -4,7 +4,17 @@ """The top-level ``services:`` block: user-declared long-running processes launched next to the job.""" # Import kinds to trigger registration. -from srtctl.services import exporters, generic, gms, infra, lmcache_server, mooncake_master, mooncake_store, ray +from srtctl.services import ( + exporters, + generic, + gms, + infra, + llm_d_sidecar, + lmcache_server, + mooncake_master, + mooncake_store, + ray, +) from srtctl.services.config import ( SERVICE_PLACEMENTS, SERVICE_STARTS, @@ -42,6 +52,7 @@ "gms", "infra", "list_service_types", + "llm_d_sidecar", "lmcache_server", "mooncake_master", "mooncake_store", diff --git a/src/srtctl/services/llm_d_sidecar.py b/src/srtctl/services/llm_d_sidecar.py new file mode 100644 index 000000000..8a321d586 --- /dev/null +++ b/src/srtctl/services/llm_d_sidecar.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``type: llm-d-sidecar``: llm-d's P/D sidecar in front of every routable decode worker. + +Implied by ``frontend.type: llm-d`` when the job has prefill and decode workers. +The Endpoint Picker routes a request to a decode endpoint and names the prefill +worker in the ``x-prefiller-host-port`` header; the sidecar on that decode +endpoint sends the prefill request first, then hands the returned +``kv_transfer_params`` to its own vLLM, whose KV connector pulls the cache from +the prefill worker. The sidecar is what the router addresses for a decode +worker, so it binds the worker's ``Process.proxy_port`` and proxies everything +else (``/metrics`` included) to the worker's HTTP port. + +Upstream: llm-d-router ``cmd/pd-sidecar`` and ``pkg/sidecar/proxy`` at v0.11.0 +(https://github.com/llm-d/llm-d-router/tree/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/sidecar/proxy). +Its ``GET /health`` answers 200 on its own, so it starts before the workers. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from marshmallow import ValidationError + +from srtctl.services.config import HttpProbe, ServiceReadinessConfig +from srtctl.services.registry import ServiceKind, ServiceLaunchContext, register_service + +if TYPE_CHECKING: + from srtctl.core.schema import SrtConfig + from srtctl.core.topology import Process + from srtctl.services.config import ServiceConfig + +LLM_D_SIDECAR_TYPE = "llm-d-sidecar" + +# The sidecar's --kv-connector protocol for the vLLM connector class the decode workers run. +# Upstream's table of protocols: pkg/sidecar/proxy/options.go (supportedKVConnectors). +SIDECAR_KV_CONNECTORS: dict[str, str] = { + "NixlConnector": "nixlv2", +} + +# Argv shown by dry-run, where there is no worker to read the ports from. +_PREVIEW_PORTS = ["--port=", "--model-server-port="] + + +def sidecar_kv_connector(backend: Any) -> str: + """The ``--kv-connector`` protocol for the decode workers' KV connector; raises ``ValueError`` when unmapped.""" + from srtctl.backends.vllm import VLLMBackend + + # The sidecar drives vLLM's KV transfer protocol; another engine's decode has no row. + connector = backend.kv_connector_class("decode") if isinstance(backend, VLLMBackend) else None + try: + return SIDECAR_KV_CONNECTORS[str(connector)] + except KeyError: + supported = ", ".join(f"{name} ({protocol})" for name, protocol in SIDECAR_KV_CONNECTORS.items()) + raise ValueError( + f"the llm-d P/D sidecar has no protocol for the {backend.type} decode KV connector {connector!r}; " + f"supported (vLLM): {supported}" + ) from None + + +@register_service(LLM_D_SIDECAR_TYPE) +class LLMDSidecarService(ServiceKind): + """llm-d P/D sidecar, one per routable decode worker; implied by the llm-d frontend.""" + + builds_command = True + default_command = ("pd-sidecar",) + default_start = "before_workers" + default_critical = True + default_placement = "decode" + default_per = "worker" + + def validate(self, service: ServiceConfig, config: SrtConfig) -> None: + from srtctl.frontends import FRONTEND_NONE, get_frontend + + proxied = ( + frozenset() + if config.frontend.type == FRONTEND_NONE + else get_frontend(config.frontend.type).proxied_worker_modes(config) + ) + if service.effective_per != "worker" or service.effective_placement not in proxied: + raise ValidationError( + f"services[{service.name}] (type {LLM_D_SIDECAR_TYPE}) fronts the workers the frontend proxies " + f"({', '.join(sorted(proxied)) or 'none for this recipe'}); set placement.node to one of them " + "and placement.per: worker" + ) + + def container_fallback(self, config: SrtConfig) -> str | None: + """The router image (``frontend.container_image``), which ships the sidecar next to the EPP.""" + return config.frontend.container_image + + def attaches_to(self, process: Process) -> bool: + return process.proxy_port is not None + + def build_command(self, service: ServiceConfig, ctx: ServiceLaunchContext) -> list[str]: + command = list(service.command) if service.command is not None else list(self.default_command) + if ctx.process is None or ctx.config is None or ctx.process.proxy_port is None: + managed = [*_PREVIEW_PORTS, "--kv-connector="] + else: + connector = sidecar_kv_connector(ctx.config.backend_for_role(ctx.process.endpoint_mode)) + managed = [ + f"--port={ctx.process.proxy_port}", + f"--model-server-port={ctx.process.http_port}", + f"--kv-connector={connector}", + ] + # The router reaches the sidecar over plain HTTP inside the job. + return [*command, *managed, "--secure-proxy=false", *service.args] + + def readiness(self, service: ServiceConfig, ctx: ServiceLaunchContext) -> ServiceReadinessConfig | None: + if ctx.process is None or ctx.process.proxy_port is None: + return None + return ServiceReadinessConfig( + http=HttpProbe(port=ctx.process.proxy_port, path="/health"), + timeout_seconds=self.default_readiness_timeout, + ) diff --git a/src/srtctl/services/registry.py b/src/srtctl/services/registry.py index 9722e105e..68b56e284 100644 --- a/src/srtctl/services/registry.py +++ b/src/srtctl/services/registry.py @@ -105,6 +105,12 @@ def template_vars(self) -> dict[str, str]: "worker_gpu_count": str(len(self.process.gpu_indices)), } ) + # The worker's own HTTP API, and the listener of the proxy in front of it + # when the frontend routes through one (Process.proxy_port). + if self.process.http_port > 0: + values["worker_http_port"] = str(self.process.http_port) + if self.process.proxy_port is not None: + values["worker_proxy_port"] = str(self.process.proxy_port) return values @@ -196,6 +202,15 @@ def wait_fleet_ready(self, service: ServiceConfig, runtime: RuntimeContext, proc also needs the head to report every member before the client starts. """ + def attaches_to(self, process: Process) -> bool: + """Whether a ``placement.per: worker`` instance of this kind attaches to ``process``. + + Every engine-0 process of the placed roles by default, followers of a + multi-node worker included; a proxy that fronts a worker's API attaches + only where that API is (``Process.proxy_port``). + """ + return True + def metrics(self, service: ServiceConfig) -> list[ServiceMetricsConfig]: """The Prometheus endpoints this service serves; empty when it publishes none. diff --git a/src/srtctl/templates/llm-d-envoy.yaml.j2 b/src/srtctl/templates/llm-d-envoy.yaml.j2 new file mode 100644 index 000000000..0878190af --- /dev/null +++ b/src/srtctl/templates/llm-d-envoy.yaml.j2 @@ -0,0 +1,94 @@ +# Envoy in front of the llm-d Endpoint Picker (frontend.type: llm-d), rendered by srtctl. +# Follows llm-d's no-Kubernetes guide: +# https://github.com/llm-d/llm-d/blob/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment/router/envoy/envoy.yaml +# Differences: the admin listener binds every interface so srtctl can probe /ready from the +# orchestrator, requests carry no timeout (agentic requests stream for minutes), and an access +# log records the endpoint the EPP picked for every request. + +admin: + address: + socket_address: {address: 0.0.0.0, port_value: {{ admin_port }}} + +static_resources: + listeners: + - name: llm-d + address: + socket_address: {address: 0.0.0.0, port_value: {{ listen_port }}} + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: llm-d + stream_idle_timeout: 0s + request_timeout: 0s + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: {{ access_log }} + log_format: + text_format_source: + inline_string: "[%START_TIME%] %REQ(:METHOD)% %REQ(:PATH)% status=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% upstream=%UPSTREAM_HOST% prefiller=%REQ(x-prefiller-host-port)% duration=%DURATION%ms\n" + route_config: + name: llm-d + virtual_hosts: + - name: llm-d + domains: ["*"] + routes: + - match: {prefix: "/"} + route: {cluster: original_destination_cluster, timeout: 0s} + http_filters: + - name: envoy.filters.http.ext_proc + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor + grpc_service: + envoy_grpc: {cluster_name: ext_proc} + timeout: 10s + processing_mode: + request_header_mode: SEND + response_header_mode: SEND + request_body_mode: FULL_DUPLEX_STREAMED + response_body_mode: FULL_DUPLEX_STREAMED + request_trailer_mode: SEND + response_trailer_mode: SEND + message_timeout: 1000s + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppress_envoy_headers: true + + clusters: + # The worker the EPP picked, named by its x-gateway-destination-endpoint header. + - name: original_destination_cluster + type: ORIGINAL_DST + connect_timeout: 1000s + lb_policy: CLUSTER_PROVIDED + # Every request holds a connection here and an ext_proc stream to the EPP for its whole + # lifetime; Envoy's default of 1024 would cap the client's concurrency. + circuit_breakers: + thresholds: + - {max_connections: 40000, max_pending_requests: 40000, max_requests: 40000} + original_dst_lb_config: + use_http_header: true + http_header_name: x-gateway-destination-endpoint + + - name: ext_proc + type: STATIC + connect_timeout: 86400s + lb_policy: LEAST_REQUEST + circuit_breakers: + thresholds: + - {max_connections: 40000, max_pending_requests: 40000, max_requests: 40000, max_retries: 1024} + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: ext_proc + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: {address: 127.0.0.1, port_value: {{ epp_port }}} diff --git a/tests/snapshots/launch/vllm__llm-d-agg.txt b/tests/snapshots/launch/vllm__llm-d-agg.txt new file mode 100644 index 000000000..3d14205a2 --- /dev/null +++ b/tests/snapshots/launch/vllm__llm-d-agg.txt @@ -0,0 +1,190 @@ +# examples/vllm/llm-d-agg.yaml +# exit_code: 0 +# srun calls: 12 + +## agg_0_node-01 +nodelist: node-01 +output: /logs/node-01_agg_w0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5200 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20000 +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6100 --device-ids 0 --gpu-memory-utilization 0.5 --max-model-len 4096 --served-model-name Qwen/Qwen3-0.6B --tensor-parallel-size 1 + +## agg_1_node-01 +nodelist: node-01 +output: /logs/node-01_agg_w1.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5201 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5401 + VLLM_PORT: 20050 +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6132 --device-ids 1 --gpu-memory-utilization 0.5 --max-model-len 4096 --served-model-name Qwen/Qwen3-0.6B --tensor-parallel-size 1 + +## llm-d-epp_0 +nodelist: node-01 +output: /logs/node-01_llm-d-epp_0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +command: | + epp --pool-name=srtctl --pool-namespace=srtctl --config-file=/logs/llm-d-epp-config.yaml --grpc-port=9002 --grpc-health-port=9003 --metrics-port=9090 --secure-serving=false + +## llm-d-envoy_0 +nodelist: node-01 +output: /logs/node-01_llm-d-envoy_0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +command: | + envoy -c /logs/llm-d-envoy.yaml --disable-hot-restart + +## service_dcgm-exporter_node-01 +nodelist: node-01 +output: /logs/service_dcgm-exporter_node-01.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-02 +nodelist: node-02 +output: /logs/service_dcgm-exporter_node-02.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-03 +nodelist: node-03 +output: /logs/service_dcgm-exporter_node-03.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_node-exporter_node-01 +nodelist: node-01 +output: /logs/service_node-exporter_node-01.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-02 +nodelist: node-02 +output: /logs/service_node-exporter_node-02.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-03 +nodelist: node-03 +output: /logs/service_node-exporter_node-03.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## tachometer +nodelist: node-01 +output: /logs/tachometer.out +srun_export_env: + POLARS_MAX_THREADS: 4 +use_bash_wrapper: False +command: | + tachometer-scraper --config /logs/tachometer_config.toml --local-dir /logs/tachometer/local --sync-interval 120 + +## +nodelist: node-01 +output: /logs/benchmark.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + SRTCTL_FRONTEND_TYPE: llm-d + SRT_FRONTEND_HOST: 127.0.0.1 + SRT_FRONTEND_PORT: 8000 +command: | + bash /srtctl-benchmarks/sa-bench/bench.sh http://localhost:8000 128 128 4x8 inf /model Qwen/Qwen3-0.6B false 8 0 0 0.8 10 2 '' true random '' false diff --git a/tests/snapshots/launch/vllm__llm-d-disagg.txt b/tests/snapshots/launch/vllm__llm-d-disagg.txt new file mode 100644 index 000000000..cf184fab1 --- /dev/null +++ b/tests/snapshots/launch/vllm__llm-d-disagg.txt @@ -0,0 +1,205 @@ +# examples/vllm/llm-d-disagg.yaml +# exit_code: 0 +# srun calls: 13 + +## service_llm-d-sidecar_decode_0_node-02 +nodelist: node-02 +output: /logs/service_llm-d-sidecar_decode_0_node-02.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + CUDA_VISIBLE_DEVICES: 0 +use_bash_wrapper: True +command: | + pd-sidecar --port=9600 --model-server-port=6100 --kv-connector=nixlv2 --secure-proxy=false + +## prefill_0_node-01 +nodelist: node-01 +output: /logs/node-01_prefill_w0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5200 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20000 +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6100 --device-ids 0 --gpu-memory-utilization 0.5 --kv-transfer-config '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' --max-model-len 4096 --served-model-name Qwen/Qwen3-0.6B --tensor-parallel-size 1 + +## decode_0_node-02 +nodelist: node-02 +output: /logs/node-02_decode_w0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5201 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5401 + VLLM_PORT: 20050 +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6100 --device-ids 0 --gpu-memory-utilization 0.5 --kv-transfer-config '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' --max-model-len 4096 --served-model-name Qwen/Qwen3-0.6B --tensor-parallel-size 1 + +## llm-d-epp_0 +nodelist: node-01 +output: /logs/node-01_llm-d-epp_0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +command: | + epp --pool-name=srtctl --pool-namespace=srtctl --config-file=/logs/llm-d-epp-config.yaml --grpc-port=9002 --grpc-health-port=9003 --metrics-port=9090 --secure-serving=false + +## llm-d-envoy_0 +nodelist: node-01 +output: /logs/node-01_llm-d-envoy_0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +command: | + envoy -c /logs/llm-d-envoy.yaml --disable-hot-restart + +## service_dcgm-exporter_node-01 +nodelist: node-01 +output: /logs/service_dcgm-exporter_node-01.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-02 +nodelist: node-02 +output: /logs/service_dcgm-exporter_node-02.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-03 +nodelist: node-03 +output: /logs/service_dcgm-exporter_node-03.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_node-exporter_node-01 +nodelist: node-01 +output: /logs/service_node-exporter_node-01.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-02 +nodelist: node-02 +output: /logs/service_node-exporter_node-02.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-03 +nodelist: node-03 +output: /logs/service_node-exporter_node-03.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## tachometer +nodelist: node-01 +output: /logs/tachometer.out +srun_export_env: + POLARS_MAX_THREADS: 4 +use_bash_wrapper: False +command: | + tachometer-scraper --config /logs/tachometer_config.toml --local-dir /logs/tachometer/local --sync-interval 120 + +## +nodelist: node-01 +output: /logs/benchmark.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + SRTCTL_FRONTEND_TYPE: llm-d + SRT_FRONTEND_HOST: 127.0.0.1 + SRT_FRONTEND_PORT: 8000 +command: | + bash /srtctl-benchmarks/sa-bench/bench.sh http://localhost:8000 128 128 4x8 inf /model Qwen/Qwen3-0.6B true 2 1 1 0.8 10 2 '' true random '' false diff --git a/tests/test_e2e.py b/tests/test_e2e.py index ecc049143..3b6650adb 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -30,7 +30,7 @@ def test_interactive_discovers_curated_examples(): assert SGLANG_ROUTER_DISAGG in examples assert MOCKER_EXAMPLE in examples - assert len(TOPOLOGY_EXAMPLES) == 22 + assert len(TOPOLOGY_EXAMPLES) == 24 # ============================================================================= diff --git a/tests/test_frontends.py b/tests/test_frontends.py index 074206f66..67b8ba686 100644 --- a/tests/test_frontends.py +++ b/tests/test_frontends.py @@ -69,6 +69,7 @@ def test_registry_lists_every_frontend_type(self): assert list_frontend_types() == [ "atomesh", "dynamo", + "llm-d", "none", "sglang", "sglang-router", @@ -222,7 +223,7 @@ def test_schema_rejects_unknown_type_at_load(self): from srtctl.core.schema import FrontendConfig, ResourceConfig, RoleConfig, SrtConfig with pytest.raises( - ValidationError, match="Unknown frontend.type 'toy-router'.*Available: atomesh, dynamo, none" + ValidationError, match="Unknown frontend.type 'toy-router'.*Available: atomesh, dynamo, llm-d, none" ): SrtConfig( name="toy", diff --git a/tests/test_llm_d_frontend.py b/tests/test_llm_d_frontend.py new file mode 100644 index 000000000..83c844031 --- /dev/null +++ b/tests/test_llm_d_frontend.py @@ -0,0 +1,315 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""llm-d router frontend (`frontend.type: llm-d`) and its P/D sidecar service.""" + +from __future__ import annotations + +import copy +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +import yaml +from marshmallow import ValidationError + +from srtctl.backends import VLLMBackend +from srtctl.core.config import load_config +from srtctl.core.schema import SrtConfig +from srtctl.core.topology import Endpoint, Process +from srtctl.frontends import LLMDFrontend, get_frontend +from srtctl.frontends.llm_d import ( + DISCOVERY_PLUGIN, + ENDPOINTS_FILE, + ENVOY_CONFIG_FILE, + EPP_CONFIG_FILE, + epp_config_document, + parse_ready_endpoints, +) +from srtctl.ports import LLM_D_ENVOY_ADMIN_PORT, LLM_D_EPP_GRPC_PORT, LLM_D_EPP_METRICS_PORT, WORKER_PROXY_PORT_BASE +from srtctl.services.implicit import effective_services +from srtctl.services.registry import ServiceLaunchContext, get_service_kind +from tests.launch_snapshots import EXAMPLES_DIR, render_launch_plan + +DISAGG = EXAMPLES_DIR / "vllm/llm-d-disagg.yaml" +AGG = EXAMPLES_DIR / "vllm/llm-d-agg.yaml" + + +def _recipe(path: Path = DISAGG) -> dict: + return yaml.safe_load(path.read_text()) + + +def _load(recipe: dict) -> SrtConfig: + return SrtConfig.Schema().load(recipe) + + +def test_registry_resolves_llm_d() -> None: + frontend = get_frontend("llm-d") + assert isinstance(frontend, LLMDFrontend) + assert frontend.required_backend == "vllm" + assert frontend.worker_api_port("decode") == "allocated" + assert frontend.frontend_metrics_port(None) == LLM_D_EPP_METRICS_PORT + + +def test_pd_proxies_decode_workers_and_implies_the_sidecar() -> None: + config = load_config(DISAGG) + assert LLMDFrontend().proxied_worker_modes(config) == frozenset({"decode"}) + sidecar = next(entry for entry in effective_services(config) if entry.service.type == "llm-d-sidecar") + assert sidecar.implicit + assert sidecar.service.effective_placement == "decode" + assert sidecar.service.effective_per == "worker" + assert sidecar.service.effective_start == "before_workers" + assert sidecar.service.effective_critical + + +def test_aggregate_job_has_no_proxy_and_no_sidecar() -> None: + config = load_config(AGG) + assert LLMDFrontend().proxied_worker_modes(config) == frozenset() + assert not any(entry.service.type == "llm-d-sidecar" for entry in effective_services(config)) + + +def test_only_routable_decode_workers_get_a_proxy_port() -> None: + """A multi-node decode worker has one API (its leader); the follower gets neither port.""" + recipe = _recipe() + recipe["roles"]["decode"] = {**recipe["roles"]["decode"], "nodes": 2, "gpus": 16} + config = _load(recipe) + endpoints = [ + Endpoint("prefill", 0, ("node0",), frozenset({0})), + Endpoint("decode", 0, ("node1", "node2"), frozenset(range(8))), + ] + processes = config.worker_processes(endpoints) + by_role = {(p.endpoint_mode, p.node_rank): p for p in processes} + assert by_role[("prefill", 0)].proxy_port is None + assert by_role[("decode", 0)].proxy_port == WORKER_PROXY_PORT_BASE + assert by_role[("decode", 1)].http_port == 0 + assert by_role[("decode", 1)].proxy_port is None + kind = get_service_kind("llm-d-sidecar") + assert [kind.attaches_to(p) for p in processes] == [False, True, False] + + +def test_sidecar_command_binds_the_proxy_port_in_front_of_the_worker() -> None: + config = load_config(DISAGG) + process = Process("node1", frozenset({0}), 7500, 6100, "decode", 0, proxy_port=9600) + service = next(e.service for e in effective_services(config) if e.service.type == "llm-d-sidecar") + ctx = ServiceLaunchContext( + runtime=MagicMock(), + node="node1", + node_ip="10.0.0.2", + node_id=0, + index=0, + role="decode", + process=process, + config=config, + ) + kind = get_service_kind("llm-d-sidecar") + assert kind.build_command(service, ctx) == [ + "pd-sidecar", + "--port=9600", + "--model-server-port=6100", + "--kv-connector=nixlv2", + "--secure-proxy=false", + ] + probe = kind.readiness(service, ctx) + assert probe is not None and probe.http is not None + assert (probe.http.port, probe.http.path) == (9600, "/health") + assert ctx.template_vars()["worker_proxy_port"] == "9600" + assert ctx.template_vars()["worker_http_port"] == "6100" + + +def test_declared_sidecar_takes_over_with_its_own_command_and_args() -> None: + recipe = _recipe() + recipe["services"] = [ + { + "name": "llm-d-sidecar", + "type": "llm-d-sidecar", + "command": ["/app/pd-sidecar"], + "args": ["--enable-prefiller-sampling"], + } + ] + config = _load(recipe) + service = next(e.service for e in effective_services(config) if e.service.type == "llm-d-sidecar") + command = get_service_kind("llm-d-sidecar").build_command(service, ServiceLaunchContext.preview()) + assert command[0] == "/app/pd-sidecar" + assert command[-1] == "--enable-prefiller-sampling" + + +def test_declared_sidecar_must_sit_on_proxied_workers() -> None: + recipe = _recipe(AGG) + recipe["services"] = [{"name": "llm-d-sidecar", "type": "llm-d-sidecar"}] + with pytest.raises(ValidationError, match="fronts the workers the frontend proxies"): + _load(recipe) + + +def test_pd_requires_an_epp_scheduler() -> None: + recipe = _recipe() + del recipe["frontend"]["epp_config"] + with pytest.raises(ValidationError, match="needs frontend.epp_config"): + _load(recipe) + + +def test_recipe_cannot_configure_discovery() -> None: + recipe = _recipe() + recipe["frontend"]["epp_config"]["dataLayer"] = {"discovery": {"pluginRef": "mine"}} + with pytest.raises(ValidationError, match="must not configure endpoint discovery"): + _load(recipe) + + +def test_decode_connector_needs_a_sidecar_protocol() -> None: + recipe = _recipe() + recipe["engine"]["connector"] = "lmcache" + with pytest.raises(ValidationError, match="no protocol for the vllm decode KV connector 'LMCacheConnectorV1'"): + _load(recipe) + + +def test_explicit_kv_transfer_config_selects_the_protocol() -> None: + """A role's own --kv-transfer-config wins over engine.connector, as on the command line.""" + recipe = _recipe() + recipe["engine"]["connector"] = None + recipe["roles"]["decode"]["args"]["kv-transfer-config"] = '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + config = _load(recipe) + assert isinstance(config.backend, VLLMBackend) + assert config.backend.kv_connector_class("decode") == "NixlConnector" + assert config.backend.kv_connector_class("prefill") is None + + +def test_one_router_replica() -> None: + recipe = _recipe() + recipe["frontend"]["enable_multiple_frontends"] = True + recipe["roles"]["decode"]["nodes"] = 1 + with pytest.raises(ValidationError, match="runs one Endpoint Picker"): + _load(recipe) + + +def test_epp_config_gets_srtctl_discovery() -> None: + user = {"plugins": [{"type": "queue-scorer"}], "dataLayer": {"sources": [{"pluginRef": "m"}]}} + original = copy.deepcopy(user) + document = epp_config_document(user, "/logs/endpoints.yaml") + assert user == original + assert list(document)[:2] == ["apiVersion", "kind"] + assert document["plugins"][-1] == { + "name": DISCOVERY_PLUGIN, + "type": "file-discovery", + "parameters": {"path": "/logs/endpoints.yaml", "watchFile": False}, + } + assert document["dataLayer"] == {"sources": [{"pluginRef": "m"}], "discovery": {"pluginRef": DISCOVERY_PLUGIN}} + + +def test_start_frontends_writes_the_router_files_and_starts_epp_then_envoy(tmp_path: Path) -> None: + config = load_config(DISAGG) + runtime = SimpleNamespace( + network_interface=None, + log_dir=tmp_path, + container_image="model.sqsh", + container_mounts={}, + environment={}, + srun_options={}, + nodes=SimpleNamespace(het_group_for=lambda node: None), + ) + processes = [ + Process("node0", frozenset({0}), 7500, 6100, "prefill", 0), + Process("node1", frozenset({0}), 7501, 6100, "decode", 0, proxy_port=9600), + ] + ips = {"node0": "10.0.0.1", "node1": "10.0.0.2"} + with ( + patch("srtctl.frontends.static_router.get_hostname_ip", side_effect=lambda node, _: ips[node]), + patch.object(LLMDFrontend, "wait_for_workers") as wait, + patch.object(LLMDFrontend, "start_process", return_value=MagicMock()) as start, + ): + managed = LLMDFrontend().start_frontends( + SimpleNamespace(frontend_nodes=["node0"], frontend_port=8000), runtime, config, config.backend, processes + ) + + # The gate probes the workers themselves (vLLM /health), not the sidecar, which answers at once. + assert [w.url for w in wait.call_args.args[0]] == ["http://10.0.0.1:6100", "http://10.0.0.2:6100"] + endpoints = yaml.safe_load((tmp_path / ENDPOINTS_FILE).read_text())["endpoints"] + assert [(e["address"], e["port"], e["labels"]["llm-d.ai/role"]) for e in endpoints] == [ + ("10.0.0.1", "6100", "prefill"), + ("10.0.0.2", "9600", "decode"), + ] + epp = yaml.safe_load((tmp_path / EPP_CONFIG_FILE).read_text()) + assert epp["dataLayer"]["discovery"] == {"pluginRef": DISCOVERY_PLUGIN} + assert {"name": "prefill", "plugins": epp["schedulingProfiles"][0]["plugins"]} == epp["schedulingProfiles"][0] + envoy = yaml.safe_load((tmp_path / ENVOY_CONFIG_FILE).read_text()) + assert envoy["admin"]["address"]["socket_address"]["port_value"] == LLM_D_ENVOY_ADMIN_PORT + listener = envoy["static_resources"]["listeners"][0]["address"]["socket_address"] + assert listener == {"address": "0.0.0.0", "port_value": 8000} + epp_cluster = envoy["static_resources"]["clusters"][1]["load_assignment"]["endpoints"][0]["lb_endpoints"][0] + assert epp_cluster["endpoint"]["address"]["socket_address"]["port_value"] == LLM_D_EPP_GRPC_PORT + + assert [p.name for p in managed] == ["llm-d-epp_0", "llm-d-envoy_0"] + assert all(p.critical for p in managed) + epp_cmd, envoy_cmd = (call.kwargs["command"] for call in start.call_args_list) + assert epp_cmd[0] == "epp" and "--config-file=/logs/llm-d-epp-config.yaml" in epp_cmd + assert "--secure-serving=false" in epp_cmd + assert envoy_cmd == ["envoy", "-c", "/logs/llm-d-envoy.yaml", "--disable-hot-restart"] + + +def test_frontend_args_cannot_move_managed_epp_flags() -> None: + config = SimpleNamespace(frontend=SimpleNamespace(args={"metrics_port": 1234})) + with pytest.raises(ValueError, match="metrics-port, which srtctl manages"): + LLMDFrontend().epp_command(config, "/logs/c.yaml") + + +def test_ready_endpoints_gauge_is_parsed() -> None: + text = ( + "# HELP llm_d_epp_ready_endpoints The number of ready endpoints.\n" + "# TYPE llm_d_epp_ready_endpoints gauge\n" + 'llm_d_epp_ready_endpoints{name="srtctl"} 2\n' + "llm_d_epp_ready_endpoints_total 9\n" + ) + assert parse_ready_endpoints(text) == 2 + assert parse_ready_endpoints("# nothing yet\n") is None + + +@pytest.mark.parametrize(("ready", "expected"), [(1, False), (2, True)]) +def test_probe_needs_envoy_ready_and_every_endpoint_scraped(ready: int, expected: bool) -> None: + def get(url: str, timeout: float) -> SimpleNamespace: + if url.endswith(f":{LLM_D_ENVOY_ADMIN_PORT}/ready"): + return SimpleNamespace(status_code=200, text="LIVE") + assert url.endswith(f":{LLM_D_EPP_METRICS_PORT}/metrics") + return SimpleNamespace(status_code=200, text=f'llm_d_epp_ready_endpoints{{name="srtctl"}} {ready}\n') + + with patch("srtctl.frontends.llm_d.requests.get", side_effect=get): + result = LLMDFrontend().probe_ready("node0", 8000, 1, 1, None) + assert result.ready is expected + assert f"{ready}/2" in result.message + + +def test_probe_waits_for_envoy() -> None: + with patch("srtctl.frontends.llm_d.requests.get", return_value=SimpleNamespace(status_code=503, text="")): + assert not LLMDFrontend().probe_ready("node0", 8000, 0, 2, None).ready + + +def test_example_launches_sidecar_workers_epp_and_envoy() -> None: + """Through the mock orchestrator: the sidecar starts before the workers, the router after them.""" + plan = render_launch_plan(DISAGG) + assert "# exit_code: 0" in plan + steps = [line[3:] for line in plan.splitlines() if line.startswith("## ")] + assert steps[:5] == [ + "service_llm-d-sidecar_decode_0_node-02", + "prefill_0_node-01", + "decode_0_node-02", + "llm-d-epp_0", + "llm-d-envoy_0", + ] + sidecar = next(line.strip() for line in plan.splitlines() if line.strip().startswith("pd-sidecar")) + decode = plan.split("## decode_0_node-02")[1].split("## ")[0] + assert f"--port={WORKER_PROXY_PORT_BASE}" in sidecar + assert "--model-server-port=6100" in sidecar and "--port 6100" in decode + assert "--kv-connector=nixlv2" in sidecar + + +def test_agg_example_launches_epp_and_envoy() -> None: + plan = render_launch_plan(AGG) + assert "# exit_code: 0" in plan + assert "## llm-d-epp_0" in plan and "## llm-d-envoy_0" in plan + assert "pd-sidecar" not in plan + + +def test_engine_must_be_vllm() -> None: + recipe = _recipe(AGG) + recipe["engine"] = "sglang" + recipe["roles"]["agg"]["args"] = {"served-model-name": "m"} + with pytest.raises(ValidationError, match="requires backend"): + _load(recipe) diff --git a/tests/test_port_allocator.py b/tests/test_port_allocator.py index e34f80130..714fd7892 100644 --- a/tests/test_port_allocator.py +++ b/tests/test_port_allocator.py @@ -24,6 +24,7 @@ MORIIO_NOTIFY_PORTS, NCCL_PORTS, NIXL_PORTS, + PROXY_PORTS, SIDECAR_GRPC_PORTS, SYS_PORTS, TOKENSPEED_PORTS, @@ -96,6 +97,7 @@ def test_every_kind_has_its_own_range(self): TRTLLM_DIST_INIT_PORTS, TOKENSPEED_PORTS, SIDECAR_GRPC_PORTS, + PROXY_PORTS, ] assert len({kind.name for kind in kinds}) == len(kinds) assert len({kind.base for kind in kinds}) == len(kinds) @@ -134,6 +136,7 @@ def test_example_topologies_bind_no_port_twice(recipe: Path): "kv_events": process.kv_events_port, "kvbm_zmq": process.kvbm_zmq_port, "sidecar_grpc": process.sidecar_grpc_port, + "proxy": process.proxy_port, "nccl": process.nccl_port, "vllm_scan": process.vllm_scan_port, "moriio_handshake": process.moriio_handshake_port, diff --git a/tests/test_services.py b/tests/test_services.py index 672b112e4..a3fe6a856 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -112,6 +112,7 @@ def test_registered_kinds() -> None: "etcd", "generic", "gms", + "llm-d-sidecar", "lmcache-server", "mooncake-master", "mooncake-store", From 90632aa58dbdc7121b08e597c1363a0dcdca72aa Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 5 Oct 2026 18:48:50 -0500 Subject: [PATCH 02/25] fix(llm-d): answer /metrics at Envoy instead of routing it to a worker A client that scrapes the public endpoint's /metrics (AIPerf does) would get whichever worker the EPP picked, a different one on every scrape. Envoy answers it with 404, as llm-d's own Slurm submission does; the workers and the EPP serve their metrics on their own listeners. Signed-off-by: Cam Quilici --- docs/llm-d.md | 3 ++- src/srtctl/templates/llm-d-envoy.yaml.j2 | 12 ++++++++++-- tests/test_llm_d_frontend.py | 3 +++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/llm-d.md b/docs/llm-d.md index 4d96c81fc..22c07b1f2 100644 --- a/docs/llm-d.md +++ b/docs/llm-d.md @@ -88,7 +88,8 @@ srtctl writes three files into the log directory (every container sees it at `/l the spelling v0.10 and v0.11 both accept. A recipe that configures discovery itself is rejected. - `llm-d-envoy.yaml`: the guide's Envoy configuration with the public port, an admin - listener srtctl can reach, no request timeout, and an access log + listener srtctl can reach, no request timeout, `/metrics` answered with 404 (routed, it + would return a different worker's counters on every scrape), and an access log (`llm-d-envoy-access.log`) recording, for every request, the endpoint the EPP picked (`upstream=`) and the prefill worker it named (`prefiller=`). diff --git a/src/srtctl/templates/llm-d-envoy.yaml.j2 b/src/srtctl/templates/llm-d-envoy.yaml.j2 index 0878190af..3ed1bacd6 100644 --- a/src/srtctl/templates/llm-d-envoy.yaml.j2 +++ b/src/srtctl/templates/llm-d-envoy.yaml.j2 @@ -2,8 +2,8 @@ # Follows llm-d's no-Kubernetes guide: # https://github.com/llm-d/llm-d/blob/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment/router/envoy/envoy.yaml # Differences: the admin listener binds every interface so srtctl can probe /ready from the -# orchestrator, requests carry no timeout (agentic requests stream for minutes), and an access -# log records the endpoint the EPP picked for every request. +# orchestrator, requests carry no timeout (agentic requests stream for minutes), /metrics is not +# routed, and an access log records the endpoint the EPP picked for every request. admin: address: @@ -36,6 +36,14 @@ static_resources: - name: llm-d domains: ["*"] routes: + # A load-balanced /metrics would be a different worker's counters on every scrape; + # the workers and the EPP serve their own. Answered here, without asking the EPP. + - match: {path: "/metrics"} + direct_response: {status: 404} + typed_per_filter_config: + envoy.filters.http.ext_proc: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExtProcPerRoute + disabled: true - match: {prefix: "/"} route: {cluster: original_destination_cluster, timeout: 0s} http_filters: diff --git a/tests/test_llm_d_frontend.py b/tests/test_llm_d_frontend.py index 83c844031..8c0b5adfe 100644 --- a/tests/test_llm_d_frontend.py +++ b/tests/test_llm_d_frontend.py @@ -234,6 +234,9 @@ def test_start_frontends_writes_the_router_files_and_starts_epp_then_envoy(tmp_p assert envoy["admin"]["address"]["socket_address"]["port_value"] == LLM_D_ENVOY_ADMIN_PORT listener = envoy["static_resources"]["listeners"][0]["address"]["socket_address"] assert listener == {"address": "0.0.0.0", "port_value": 8000} + routes = envoy["static_resources"]["listeners"][0]["filter_chains"][0]["filters"][0]["typed_config"]["route_config"] + metrics_route = routes["virtual_hosts"][0]["routes"][0] + assert metrics_route["match"] == {"path": "/metrics"} and metrics_route["direct_response"] == {"status": 404} epp_cluster = envoy["static_resources"]["clusters"][1]["load_assignment"]["endpoints"][0]["lb_endpoints"][0] assert epp_cluster["endpoint"]["address"]["socket_address"]["port_value"] == LLM_D_EPP_GRPC_PORT From f847fc73414df512eb2330804eb336c905e955cb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 5 Oct 2026 21:33:43 -0500 Subject: [PATCH 03/25] chore(llm-d): add NVIDIA SPDX headers to llm-d files Signed-off-by: Cam Quilici --- src/srtctl/frontends/llm_d.py | 1 + src/srtctl/services/llm_d_sidecar.py | 1 + src/srtctl/templates/llm-d-envoy.yaml.j2 | 3 +++ 3 files changed, 5 insertions(+) diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py index 3af2af24e..7c3c52d2c 100644 --- a/src/srtctl/frontends/llm_d.py +++ b/src/srtctl/frontends/llm_d.py @@ -1,3 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. # SPDX-License-Identifier: Apache-2.0 diff --git a/src/srtctl/services/llm_d_sidecar.py b/src/srtctl/services/llm_d_sidecar.py index 8a321d586..8a354e662 100644 --- a/src/srtctl/services/llm_d_sidecar.py +++ b/src/srtctl/services/llm_d_sidecar.py @@ -1,3 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. # SPDX-License-Identifier: Apache-2.0 diff --git a/src/srtctl/templates/llm-d-envoy.yaml.j2 b/src/srtctl/templates/llm-d-envoy.yaml.j2 index 3ed1bacd6..98b257ed5 100644 --- a/src/srtctl/templates/llm-d-envoy.yaml.j2 +++ b/src/srtctl/templates/llm-d-envoy.yaml.j2 @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 # Envoy in front of the llm-d Endpoint Picker (frontend.type: llm-d), rendered by srtctl. # Follows llm-d's no-Kubernetes guide: # https://github.com/llm-d/llm-d/blob/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment/router/envoy/envoy.yaml From ab0b08f8f02d4f714450c0662b285c630a0b1d22 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 6 Oct 2026 18:20:52 -0500 Subject: [PATCH 04/25] feat(llm-d): route to DP ranks and by the workers' KV-cache events data-parallel-external-lb launches every vLLM DP rank as its own vllm serve that the EPP addresses. roles..kv_events connects the workers' KV-cache event publishers to the EPP's precise prefix-cache producer, with srtctl wiring the event socket, topics, and the token producer's render URL. Signed-off-by: Cam Quilici --- docs/engines.md | 2 +- docs/llm-d.md | 83 ++- examples/README.md | 2 +- examples/vllm/llm-d-dp-ranks.yaml | 75 +++ src/srtctl/backends/vllm.py | 80 ++- src/srtctl/cli/mixins/service_stage.py | 5 +- src/srtctl/cli/mixins/worker_stage.py | 15 +- src/srtctl/core/supervisor.py | 7 +- src/srtctl/core/topology.py | 12 +- src/srtctl/frontends/base.py | 11 + src/srtctl/frontends/llm_d.py | 144 ++++- src/srtctl/ports.py | 3 + .../snapshots/launch/vllm__llm-d-dp-ranks.txt | 572 ++++++++++++++++++ tests/test_design_rules.py | 3 + tests/test_e2e.py | 2 +- tests/test_llm_d_frontend.py | 182 +++++- 16 files changed, 1170 insertions(+), 28 deletions(-) create mode 100644 examples/vllm/llm-d-dp-ranks.yaml create mode 100644 tests/snapshots/launch/vllm__llm-d-dp-ranks.txt diff --git a/docs/engines.md b/docs/engines.md index 68a333f5b..f29c742ae 100644 --- a/docs/engines.md +++ b/docs/engines.md @@ -144,7 +144,7 @@ Set `engine.dp_launch_mode: per_gpu` only when temporarily preserving the legacy When `TP x PP` fits on one node, srtslurm derives `--data-parallel-size-local` and `--data-parallel-start-rank`, then enables `--data-parallel-hybrid-lb` so every node-local process registers with the Dynamo frontend. When `TP x PP` is larger than the node-local GPU allocation, srtslurm instead derives the multi-node rendezvous arguments and makes every process except the global leader headless. For example, both DP4 x TP4 and DP2 x TP8 are selected automatically on four-GPU nodes. -Do not set `data-parallel-size-local`, `data-parallel-start-rank`, `data-parallel-hybrid-lb`, or `headless` manually; srtslurm owns those values. The allocation must be regular: `DP x TP x PP` must match the endpoint GPU count, and a TP/PP replica must divide evenly within or across nodes. +Do not set `data-parallel-size-local`, `data-parallel-start-rank`, `data-parallel-hybrid-lb`, or `headless` manually; srtslurm owns those values. Behind a router frontend, `data-parallel-external-lb: true` instead launches every DP rank as its own `vllm serve --data-parallel-rank ` on its own HTTP port, so the router addresses each rank ([llm-d](llm-d.md#routing-to-dp-ranks)). The allocation must be regular: `DP x TP x PP` must match the endpoint GPU count, and a TP/PP replica must divide evenly within or across nodes. ### TRT-LLM metrics publication diff --git a/docs/llm-d.md b/docs/llm-d.md index 22c07b1f2..38a15e400 100644 --- a/docs/llm-d.md +++ b/docs/llm-d.md @@ -146,9 +146,83 @@ the EPP's built-in default profile. See [`examples/vllm/llm-d-disagg.yaml`](https://github.com/NVIDIA/srt-slurm/blob/main/examples/vllm/llm-d-disagg.yaml) and [`examples/vllm/llm-d-agg.yaml`](https://github.com/NVIDIA/srt-slurm/blob/main/examples/vllm/llm-d-agg.yaml). +## Routing to DP ranks + A vLLM worker with `data-parallel-size` is one `vllm serve` that balances its DP ranks itself (the leader serves the API, other nodes run headless ranks), so it is one EPP -endpoint, as it is one SMG worker. +endpoint, as it is one SMG worker. Set `data-parallel-external-lb: true` in the role's +`args` to have the EPP route to the ranks themselves, as llm-d's wide-EP guide does: srtctl +then launches every DP rank as its own `vllm serve --data-parallel-rank +--data-parallel-address ` (vLLM's external load balancing) on its own +allocated HTTP port, on one node or across several, and lists each rank in the endpoints +file. Each rank's step and log carry `_dp` (`agg_0__dp3`, `_agg_w0_dp3.out`), +and in a prefill/decode job every decode rank gets its own P/D sidecar. vLLM accepts +external load balancing for MoE models only. See +[`examples/vllm/llm-d-dp-ranks.yaml`](https://github.com/NVIDIA/srt-slurm/blob/main/examples/vllm/llm-d-dp-ranks.yaml). + +```yaml +roles: + agg: + gpus: 16 + args: + data-parallel-size: 16 + data-parallel-external-lb: true +``` + +## Precise prefix-cache routing + +The approximate prefix scorers guess where a prefix is cached from the requests the EPP +routed. With `roles..kv_events` the workers report it instead: every vLLM worker of +the role publishes its KV-cache events (blocks stored and evicted) to the EPP's +[`precise-prefix-cache-producer`](https://github.com/llm-d/llm-d-router/tree/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/epp/framework/plugins/requestcontrol/dataproducer/preciseprefixcache), +which indexes them per endpoint; `prefix-cache-scorer` with +`prefixMatchInfoProducerName: precise-prefix-cache-producer` scores the endpoints from +that index. The producer hashes the prompt's real token IDs, so the configuration also +needs a `token-producer`, which tokenizes through a worker's `/v1/*/render` endpoints: + +```yaml +frontend: + epp_config: + plugins: + - type: token-producer + - type: precise-prefix-cache-producer + parameters: + indexerConfig: + kvBlockIndexConfig: + enableMetrics: true # index admissions and lookup hits on /metrics + - type: prefix-cache-scorer + parameters: + prefixMatchInfoProducerName: precise-prefix-cache-producer + - type: max-score-picker + schedulingProfiles: [...] +roles: + agg: + kv_events: true +``` + +srtctl wires the rest: + +- The producer's `kvEventsConfig.zmqEndpoint` is `tcp://*:5557` (`discoverPods: false`): + the EPP binds one ZMQ socket on the router node and every worker connects to it. The + producer's per-pod mode dials the same port on every endpoint address, which workers + sharing a node cannot all bind (llm-d-router v0.11's file discovery gives every + endpoint rank 0). +- Every publishing worker gets `--kv-events-config` with `endpoint: + tcp://:5557` and `topic: kv@
:@`, naming the + endpoint the EPP routes to (a decode worker's sidecar port) and the model requests + name; the producer files the events under that endpoint + ([`vllm_adapter.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/kvevents/engineadapter/vllm_adapter.go#L52-L58)). + vLLM adds a publisher's DP rank to its port, so an external-LB rank `r` is given + `5557 - r`. +- The token producer's `vllm.url` is the first prefill worker's (else the first worker's) + own vLLM API, and `modelName` defaults to the served model name. + +The roles that publish and the producer go together: either without the other is +rejected, as are a producer `kvEventsConfig` that sets the socket or discovery, a +`token-producer` `vllm.url`, and a router not on the head node. A role with +`data-parallel-size` publishes only with `data-parallel-external-lb: true`: each DP rank +publishes its own events, and the EPP can place a prefix only on an endpoint that is +that rank. ## Limitations @@ -158,6 +232,7 @@ endpoint, as it is one SMG worker. rejected. - The endpoints file is written once (`watchFile: false`) and the EPP does not eject a worker that fails; a crashed worker or sidecar fails the job instead. -- The EPP routes to a worker, not to one of its DP ranks. -- Precise prefix-cache routing (KV events into the EPP) is not wired up; use the - approximate prefix scorers. +- vLLM's `data-parallel-multi-port-external-lb` (one supervisor per node serving a port + per rank) is not used: its ranks publish KV events on per-rank ports that file + discovery can only reach with llm-d-router's per-endpoint `rankIndex` (after v0.11.0). +- The precise prefix index lives in the one EPP; KV events go to that EPP alone. diff --git a/examples/README.md b/examples/README.md index dd925a792..c936e4d26 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,7 +7,7 @@ Small starting points, one per frontend and topology. The matrix examples serve | Backend | Dynamo frontend | Native router | Router-free direct | | --- | --- | --- | --- | | SGLang | `sglang/dynamo-agg.yaml`, `sglang/dynamo-disagg.yaml` | `sglang/sglang-router-agg.yaml`, `sglang/sglang-router-disagg.yaml`, `sglang/smg-disagg.yaml` (SMG) | `sglang/sglang-direct-agg.yaml` | -| vLLM | `vllm/dynamo-agg.yaml`, `vllm/dynamo-disagg.yaml` | `vllm/vllm-router-agg.yaml`, `vllm/vllm-router-disagg.yaml`, `vllm/vllm-router-moriio-disagg.yaml` (ROCm, MoRI-IO discovery), `vllm/smg-agg.yaml`, `vllm/smg-dep16.yaml`, `vllm/smg-disagg-grpc.yaml` (SMG; P/D over gRPC with NIXL), `vllm/llm-d-agg.yaml`, `vllm/llm-d-disagg.yaml` (llm-d EPP behind Envoy; P/D through the llm-d sidecar) | `vllm/vllm-direct-agg.yaml` | +| vLLM | `vllm/dynamo-agg.yaml`, `vllm/dynamo-disagg.yaml` | `vllm/vllm-router-agg.yaml`, `vllm/vllm-router-disagg.yaml`, `vllm/vllm-router-moriio-disagg.yaml` (ROCm, MoRI-IO discovery), `vllm/smg-agg.yaml`, `vllm/smg-dep16.yaml`, `vllm/smg-disagg-grpc.yaml` (SMG; P/D over gRPC with NIXL), `vllm/llm-d-agg.yaml`, `vllm/llm-d-disagg.yaml` (llm-d EPP behind Envoy; P/D through the llm-d sidecar), `vllm/llm-d-dp-ranks.yaml` (llm-d routing to every external-LB DP rank by its KV-cache events) | `vllm/vllm-direct-agg.yaml` | | TRT-LLM | `trtllm/dynamo-agg.yaml`, `trtllm/dynamo-disagg.yaml`, `trtllm/dynamo-disagg-multinode.yaml` (TP16 decode across two 8-GPU nodes: one srun, one host per rank under `--distribution=arbitrary`) | `trtllm/trtllm-serve-disagg.yaml`, `trtllm/smg-agg.yaml` (SMG) | `trtllm/trtllm-serve-agg.yaml` | | TokenSpeed | `tokenspeed/dynamo-agg.yaml`, `tokenspeed/dynamo-disagg.yaml` | `tokenspeed/smg-agg.yaml`, `tokenspeed/smg-disagg.yaml` (SMG, gRPC engines) | | | Mocker | `mocker/dynamo-agg.yaml` | | | diff --git a/examples/vllm/llm-d-dp-ranks.yaml b/examples/vllm/llm-d-dp-ranks.yaml new file mode 100644 index 000000000..4ef980b27 --- /dev/null +++ b/examples/vllm/llm-d-dp-ranks.yaml @@ -0,0 +1,75 @@ +# vLLM data parallel 16 with expert parallelism across two nodes, behind the llm-d +# router, which routes to every DP rank by the KV-cache blocks it holds (docs/llm-d.md). +# +# `data-parallel-external-lb` makes every DP rank its own `vllm serve +# --data-parallel-rank` on its own port, so the endpoints file lists 16 endpoints. +# `kv_events` makes every rank publish its KV-cache events to the Endpoint Picker +# (EPP), whose precise prefix-cache producer indexes them; the token producer +# tokenizes each prompt through a worker's render endpoint. srtctl wires the +# event socket, the topics, and the render URL. +# +# Aliases (`llm-d-vllm`) resolve through srtslurm.yaml; see examples/README.md. + +schema: 2 +name: "qwen3-30b-a3b-vllm-llm-d-dp-ranks" + +slurm: + time_limit: "00:30:00" + +model: + path: "hf:Qwen/Qwen3-30B-A3B" + container: "llm-d-vllm" + precision: "bf16" + +resources: + gpu_type: "h100" + gpus_per_node: 8 +frontend: + type: llm-d + enable_multiple_frontends: false + epp_config: + plugins: + # Tokenizes each prompt through a worker's render endpoint (srtctl sets vllm.url and modelName). + - type: token-producer + # Indexes the ranks' KV-cache events (srtctl sets kvEventsConfig's socket). + - type: precise-prefix-cache-producer + parameters: + indexerConfig: + kvBlockIndexConfig: + enableMetrics: true + - type: prefix-cache-scorer + parameters: + prefixMatchInfoProducerName: precise-prefix-cache-producer + - type: queue-scorer + - type: max-score-picker + schedulingProfiles: + - name: default + plugins: + - pluginRef: prefix-cache-scorer + weight: 3 + - pluginRef: queue-scorer + weight: 2 + - pluginRef: max-score-picker + +engine: vllm +roles: + agg: + nodes: 2 + workers: 1 + gpus: 16 + kv_events: true + env: + PYTHONUNBUFFERED: "1" + args: + served-model-name: "Qwen/Qwen3-30B-A3B" + tensor-parallel-size: 1 + data-parallel-size: 16 + data-parallel-external-lb: true + enable-expert-parallel: true + max-model-len: 4096 + +benchmark: + type: "sa-bench" + isl: 128 + osl: 128 + concurrencies: "16x32" diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index f0c959379..6874926c9 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -690,6 +690,18 @@ def _is_dp_mode(self, mode: WorkerMode) -> bool: dp_size = self._get_dp_size(mode) return dp_size is not None and int(dp_size) > 1 + def is_external_lb(self, mode: WorkerMode) -> bool: + """Whether a DP mode's ranks are each an API server of their own (``data-parallel-external-lb``). + + A router then addresses every rank: each is its own ``vllm serve + --data-parallel-rank`` process on its own HTTP port (vLLM's external load + balancing) instead of one server that balances its ranks itself. + """ + return self._is_dp_mode(mode) and any( + normalize_vllm_config_key(key) == "data-parallel-external-lb" and value + for key, value in self.get_config_for_mode(mode).items() + ) + def _get_dp_size(self, mode: WorkerMode) -> int | None: """Get the data-parallel-size for a mode, or None if not in DP mode.""" config = self.get_config_for_mode(mode) @@ -908,10 +920,22 @@ def endpoints_to_processes( frontend = get_frontend(frontend_type) if frontend.worker_launch == "direct" and not frontend.expands_node_local_dp: # One `vllm serve` owns every DP rank of the endpoint, and only the - # leader serves the API, so the standard topology applies. - processes = endpoints_to_processes( - endpoints, port_allocator=allocator, sidecar_grpc=dynamo_sidecar, bootstrap_ports=False - ) + # leader serves the API, so the standard topology applies; behind a + # router, an external-LB endpoint is one `vllm serve` per rank instead. + processes = [] + for endpoint in endpoints: + if self.is_external_lb(endpoint.mode) and frontend.worker_api_port(endpoint.mode) == "allocated": + processes.extend( + self._dp_per_gpu_endpoints_to_processes( + [endpoint], allocator, sidecar_grpc=dynamo_sidecar, every_rank_serves=True + ) + ) + else: + processes.extend( + endpoints_to_processes( + [endpoint], port_allocator=allocator, sidecar_grpc=dynamo_sidecar, bootstrap_ports=False + ) + ) elif not any(self._is_dp_mode(ep.mode) for ep in endpoints): # Standard TP mode: one process per node, or one per engine of the # worker under backend.failover (engine 0 plus its shadows). @@ -954,8 +978,13 @@ def _dp_per_gpu_endpoints_to_processes( allocator: NodePortAllocator, *, sidecar_grpc: bool, + every_rank_serves: bool = False, ) -> list[Process]: - """DP+EP mode with one process per DP rank (TP x PP GPUs each).""" + """DP+EP mode with one process per DP rank (TP x PP GPUs each). + + Only rank 0 serves the API unless ``every_rank_serves`` (external load + balancing), when every rank gets its own HTTP port. + """ from srtctl.core.topology import Process processes: list[Process] = [] @@ -993,16 +1022,17 @@ def _dp_per_gpu_endpoints_to_processes( nixl_base_port = allocator.next(NIXL_PORTS, size=dp_size) for node in endpoint.nodes: for rank_gpus in rank_gpu_groups: - is_leader = dp_rank == 0 + serves_api = dp_rank == 0 or every_rank_serves processes.append( Process( node=node, gpu_indices=rank_gpus, sys_port=allocator.next(SYS_PORTS), - http_port=allocator.next(HTTP_PORTS, node) if is_leader else 0, + http_port=allocator.next(HTTP_PORTS, node) if serves_api else 0, endpoint_mode=endpoint.mode, endpoint_index=endpoint.index, node_rank=dp_rank, # dp_rank stored in node_rank for now + dp_rank=dp_rank, kv_events_port=allocator.next(KV_EVENTS_PORTS), nixl_port=nixl_base_port, dp_rpc_port=dp_rpc_port, @@ -1215,18 +1245,41 @@ def build_worker_command( and not spans_nodes ) + # Behind a router, an external-LB rank is a `vllm serve` of its own (endpoints_to_processes). + is_rank_server = self.is_external_lb(mode) and not binds_public_port and not frontend.expands_node_local_dp + if frontend.expands_node_local_dp and is_dp_mode and self.dp_launch_mode != "per_node": raise ValueError( f"frontend.type: {frontend.type} with data-parallel-size requires backend.dp_launch_mode: per_node" ) - if node_rank == 0 or is_router_local_dp: + if node_rank == 0 or is_router_local_dp or is_rank_server: api_port = runtime.frontend_port if binds_public_port else process.http_port cmd.extend(["--host", "0.0.0.0", "--port", str(api_port)]) srtslurm_owned["host"] = "0.0.0.0" srtslurm_owned["port"] = str(api_port) - if is_router_local_dp: + if is_rank_server: + for key in list(config): + if normalize_vllm_config_key(key) in { + "data-parallel-rank", + "data-parallel-size-local", + "data-parallel-start-rank", + "data-parallel-address", + "data-parallel-rpc-port", + }: + config.pop(key) + cmd.extend( + [ + "--data-parallel-rank", + str(process.dp_rank), + "--data-parallel-address", + leader_ip, + "--data-parallel-rpc-port", + str(process.dp_rpc_port or VLLM_DATA_PARALLEL_RPC_PORT), + ] + ) + elif is_router_local_dp: for key in list(config): if normalize_vllm_config_key(key) in { "data-parallel-size-local", @@ -1297,6 +1350,15 @@ def build_worker_command( device_ids = ",".join(str(i) for i in sorted(process.gpu_indices)) if device_ids: cmd.extend(["--device-ids", device_ids]) + kv_events = self.get_kv_events_config_for_mode(mode) + subscriber = frontend.kv_events_subscriber(process, runtime, served_model_name) if kv_events else None + if kv_events and subscriber is not None: + host, port, topic = subscriber + # vLLM adds the publisher's DP rank to the port it is given; start that far + # below so every rank connects to the one subscriber. + rank = process.dp_rank or 0 + kv_events = {**kv_events, "endpoint": f"tcp://{host}:{port - rank}", "topic": topic} + cmd.extend(["--kv-events-config", json.dumps(kv_events)]) cmd.extend(_config_to_cli_args(config)) return cmd diff --git a/src/srtctl/cli/mixins/service_stage.py b/src/srtctl/cli/mixins/service_stage.py index 5bcf74d2d..fa0799f1f 100644 --- a/src/srtctl/cli/mixins/service_stage.py +++ b/src/srtctl/cli/mixins/service_stage.py @@ -83,10 +83,11 @@ def service_step_name(service: ServiceConfig, node: str, instances: int, process """Slurm step name (and log stem) for one instance of a service. ``service_`` alone for a single instance, ``service__`` per - node, ``service____`` for an instance attached to a worker. + node, ``service____`` for an instance attached to a worker + (plus ``_dp`` when it attaches to one DP rank of a per-rank worker). """ if process is not None: - return f"service_{service.name}_{process.endpoint_mode}_{process.endpoint_index}_{node}" + return f"service_{service.name}_{process.endpoint_mode}_{process.endpoint_index}_{node}{process.engine_suffix}" suffix = f"_{node}" if instances > 1 else "" return f"service_{service.name}{suffix}" diff --git a/src/srtctl/cli/mixins/worker_stage.py b/src/srtctl/cli/mixins/worker_stage.py index 9e6bb9018..983602f71 100644 --- a/src/srtctl/cli/mixins/worker_stage.py +++ b/src/srtctl/cli/mixins/worker_stage.py @@ -39,15 +39,20 @@ WORKER_TERMINATE_TIMEOUT_SECONDS = 30.0 -def worker_step_name(mode: str, index: int, node: str, attempt: int = 0, engine_id: int = 0) -> str: +def worker_step_name( + mode: str, index: int, node: str, attempt: int = 0, engine_id: int = 0, dp_rank: int | None = None +) -> str: """The Slurm step name (and registry name) of a worker step. - ``engine_id`` > 0 is a shadow engine of the worker (vLLM ``engine.failover``) - and adds ``_e``. ``attempt`` is the supervisor's relaunch count; a + ``dp_rank`` names one rank of a per-rank endpoint (``_dp``). ``engine_id`` + > 0 is a shadow engine of the worker (vLLM ``engine.failover``) and adds + ``_e``. ``attempt`` is the supervisor's relaunch count; a relaunched step carries an ``_r`` suffix so ``squeue --steps`` can never confuse it with a lingering step of the life it replaces. """ name = f"{mode}_{index}_{node}" + if dp_rank is not None: + name = f"{name}_dp{dp_rank}" if engine_id: name = f"{name}_e{engine_id}" return f"{name}_r{attempt}" if attempt else name @@ -430,7 +435,9 @@ def __missing__(self, key: str) -> str: endpoint_nodes = {endpoint_process.node for endpoint_process in endpoint_processes} env_to_unset = ["VLLM_PORT"] if backend.type == "vllm" and len(endpoint_nodes) > 1 else None - step_name = worker_step_name(mode, index, process.node, attempt, getattr(process, "engine_id", 0)) + step_name = worker_step_name( + mode, index, process.node, attempt, getattr(process, "engine_id", 0), getattr(process, "dp_rank", None) + ) proc = start_srun_process( command=cmd, nodelist=[process.node], diff --git a/src/srtctl/core/supervisor.py b/src/srtctl/core/supervisor.py index 38afdf0ca..3fea8078e 100644 --- a/src/srtctl/core/supervisor.py +++ b/src/srtctl/core/supervisor.py @@ -190,7 +190,12 @@ def track( policy = policy_for(mode) if not policy.enabled: continue - expected = {worker_step_name(mode, index, process.node, engine_id=engine) for process in processes} + expected = { + worker_step_name( + mode, index, process.node, engine_id=engine, dp_rank=getattr(process, "dp_rank", None) + ) + for process in processes + } mine = sorted(names & expected) if not mine: logger.warning("Worker supervisor: no registered steps found for %s", _label(key)) diff --git a/src/srtctl/core/topology.py b/src/srtctl/core/topology.py index eb05962e7..addbfd16e 100644 --- a/src/srtctl/core/topology.py +++ b/src/srtctl/core/topology.py @@ -156,6 +156,8 @@ class Process: endpoint_mode: The mode of the parent endpoint endpoint_index: The index of the parent endpoint node_rank: Rank within the endpoint (0 for leader) + dp_rank: The DP rank this process runs when the endpoint is one process per rank + (vLLM ``dp_launch_mode: per_gpu`` or external load balancing); None otherwise engine_id: Which engine of the worker this is. 0 is the one every job has; under ``backend.failover`` (vLLM shadow engine recovery) engines 1.. are the standbys, sharing node, GPUs and node_rank with engine 0 but with @@ -191,6 +193,7 @@ class Process: # Inherited from the parent Endpoint when the job is heterogeneous. het_group: int | None = None engine_id: int = 0 + dp_rank: int | None = None kvbm_zmq_port: int | None = None sidecar_grpc_port: int | None = None proxy_port: int | None = None @@ -210,8 +213,13 @@ def is_leader(self) -> bool: @property def engine_suffix(self) -> str: - """Step-name and log-name suffix that tells a shadow engine apart from engine 0 (``""`` for it).""" - return f"_e{self.engine_id}" if self.engine_id else "" + """Step-name and log-name suffix that tells a worker's processes on one node apart. + + ``_dp`` for one DP rank of a per-rank endpoint, ``_e`` for a shadow + engine; ``""`` for the only process. + """ + rank = f"_dp{self.dp_rank}" if self.dp_rank is not None else "" + return f"{rank}_e{self.engine_id}" if self.engine_id else rank @property def cuda_visible_devices(self) -> str: diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py index 94fdd3245..a0805ca5a 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py @@ -189,6 +189,17 @@ def proxied_worker_modes(self, config: Any) -> frozenset[str]: """ return frozenset() + def kv_events_subscriber( + self, process: "Process", runtime: "RuntimeContext", model_name: str + ) -> tuple[str, int, str] | None: + """Where a worker publishes its KV-cache events for this router: ``(host, port, topic)``. + + ``None`` when the router subscribes to none; a role's ``kv_events`` then + configures nothing on a direct worker. The engine connects its publisher + to ``host:port`` (adjusting for any per-rank port offset of its own). + """ + return None + def frontend_metrics_port(self, frontend_args: dict[str, Any] | None) -> int | None: """Port of a Prometheus listener separate from the routing port, or ``None`` when metrics share it.""" return None diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py index 7c3c52d2c..10649a19f 100644 --- a/src/srtctl/frontends/llm_d.py +++ b/src/srtctl/frontends/llm_d.py @@ -47,6 +47,7 @@ LLM_D_ENVOY_ADMIN_PORT, LLM_D_EPP_GRPC_PORT, LLM_D_EPP_HEALTH_PORT, + LLM_D_EPP_KV_EVENTS_PORT, LLM_D_EPP_METRICS_PORT, ) from srtctl.services.config import ServiceConfig, ServicePlacementConfig @@ -74,6 +75,15 @@ # Gauge of endpoints whose metrics the EPP scraped within its staleness window # (pkg/epp/metrics/llm_d_router_metrics.go). READY_ENDPOINTS_METRIC = "llm_d_epp_ready_endpoints" +# Precise prefix-cache routing: the producer indexes the workers' KV-cache events, which +# it receives on one ZMQ socket srtctl owns; the token producer tokenizes each prompt the +# way the engine does, through a worker's /v1/*/render endpoints +# (pkg/epp/framework/plugins/requestcontrol/dataproducer/{preciseprefixcache,tokenizer}). +PRECISE_PREFIX_PRODUCER = "precise-prefix-cache-producer" +TOKEN_PRODUCER = "token-producer" +_MANAGED_KV_EVENTS_KEYS = frozenset({"zmqEndpoint", "discoverPods", "podDiscoveryConfig"}) +# Token-producer backends other than vLLM's render endpoints. +_OTHER_TOKENIZERS = frozenset({"estimate", "udsTokenizerConfig"}) # EPP flags srtctl sets; frontend.args may not repeat them. _MANAGED_EPP_FLAGS = frozenset( {"pool-name", "pool-namespace", "config-file", "config-text", "grpc-port", "grpc-health-port", "metrics-port"} @@ -85,6 +95,11 @@ def _is_pd(config: Any) -> bool: return topology.num_prefill > 0 and topology.num_decode > 0 +def _routed_port(process: Process) -> int: + """The port the EPP routes a worker's traffic to: its proxy's when it has one, else its own.""" + return process.proxy_port if process.proxy_port is not None else process.http_port + + def _container_path(name: str) -> str: """``log_dir/`` as every container sees it (``log_dir`` is mounted at ``/logs``).""" return str(Path("/logs") / name) @@ -111,6 +126,44 @@ def epp_config_document(epp_config: dict[str, Any] | None, endpoints_path: str) return document +def _plugins(epp_config: dict[str, Any] | None, plugin_type: str) -> list[dict[str, Any]]: + return [plugin for plugin in (epp_config or {}).get("plugins") or [] if plugin.get("type") == plugin_type] + + +def _parameters(plugin: dict[str, Any]) -> dict[str, Any]: + return plugin.get("parameters") or {} + + +def _vllm_token_producer(plugin: dict[str, Any]) -> bool: + return not _OTHER_TOKENIZERS & _parameters(plugin).keys() + + +def with_kv_events(document: dict[str, Any], render_url: str, model_name: str) -> dict[str, Any]: + """Point the precise prefix-cache producer at srtctl's KV-event socket and the token producer at a worker. + + The EPP binds the socket and every publishing worker connects to it (the + producer's global-socket mode, ``kvEventsConfig.zmqEndpoint``): its per-pod + mode dials one fixed port per endpoint address, which workers sharing a node + cannot all bind. vLLM token producers render through ``render_url``. + """ + for plugin in document.get("plugins") or []: + parameters = _parameters(plugin) + if plugin.get("type") == PRECISE_PREFIX_PRODUCER: + kv_events = parameters.get("kvEventsConfig") or {} + parameters["kvEventsConfig"] = { + **kv_events, + "zmqEndpoint": f"tcp://*:{LLM_D_EPP_KV_EVENTS_PORT}", + "discoverPods": False, + } + elif plugin.get("type") == TOKEN_PRODUCER and _vllm_token_producer(plugin): + parameters.setdefault("modelName", model_name) + parameters["vllm"] = {**(parameters.get("vllm") or {}), "url": render_url} + else: + continue + plugin["parameters"] = parameters + return document + + def envoy_config(listen_port: int, access_log: str) -> str: """Envoy in front of the EPP: ext_proc to it on localhost, then ORIGINAL_DST to the endpoint it picked.""" from jinja2 import Environment, FileSystemLoader @@ -174,6 +227,7 @@ def validate(self, config: Any) -> None: "frontend.epp_config must not configure endpoint discovery: srtctl adds the file-discovery " "plugin and dataLayer.discovery for the workers it launches" ) + self._validate_kv_events(config) if _is_pd(config): if not epp_config.get("schedulingProfiles"): raise ValueError( @@ -182,6 +236,81 @@ def validate(self, config: Any) -> None: ) sidecar_kv_connector(config.backend_for_role("decode")) + def _validate_kv_events(self, config: Any) -> None: + """Workers publish KV-cache events exactly when the EPP has a precise prefix-cache producer to index them.""" + epp_config = config.frontend.epp_config + topology = config.topology + publishing = [ + mode + for mode, count in ( + ("prefill", topology.num_prefill), + ("decode", topology.num_decode), + ("agg", topology.num_agg), + ) + if count and config.backend_for_role(mode).get_kv_events_config_for_mode(mode) + ] + producers = _plugins(epp_config, PRECISE_PREFIX_PRODUCER) + if not producers: + if publishing: + raise ValueError( + f"roles.{publishing[0]}.kv_events publishes KV-cache events for the EPP to index; " + f"add a {PRECISE_PREFIX_PRODUCER} plugin to frontend.epp_config" + ) + return + if not publishing: + raise ValueError( + f"frontend.epp_config's {PRECISE_PREFIX_PRODUCER} indexes the workers' KV-cache events; " + "set roles..kv_events on the roles it scores" + ) + for producer in producers: + managed = _MANAGED_KV_EVENTS_KEYS & (_parameters(producer).get("kvEventsConfig") or {}).keys() + if managed: + raise ValueError( + f"frontend.epp_config sets {PRECISE_PREFIX_PRODUCER} kvEventsConfig.{', '.join(sorted(managed))}, " + "which srtctl manages: the EPP binds one KV-event socket the workers connect to" + ) + token_producers = [plugin for plugin in _plugins(epp_config, TOKEN_PRODUCER) if _vllm_token_producer(plugin)] + if not token_producers: + raise ValueError( + f"{PRECISE_PREFIX_PRODUCER} needs the engine's token IDs: add a {TOKEN_PRODUCER} plugin " + "to frontend.epp_config (srtctl points it at a worker's render endpoint)" + ) + if any("url" in (_parameters(plugin).get("vllm") or {}) for plugin in token_producers): + raise ValueError(f"frontend.epp_config sets {TOKEN_PRODUCER} vllm.url, which srtctl points at a worker") + frontend = config.frontend + if ( + frontend.enable_multiple_frontends and config.engine_node_count > 1 + ) or frontend.placement.location != "head": + raise ValueError( + "roles..kv_events: the workers publish to the EPP on the head node; set " + "frontend.enable_multiple_frontends: false and keep frontend.placement.node: head" + ) + for mode in publishing: + backend = config.backend_for_role(mode) + if backend._is_dp_mode(mode) and not backend.is_external_lb(mode): + raise ValueError( + f"roles.{mode}.kv_events with data-parallel-size: vLLM publishes every DP rank's events on its " + "own port, so the EPP must route to the ranks themselves; set " + f"roles.{mode}.args.data-parallel-external-lb: true" + ) + + def kv_events_subscriber(self, process: Process, runtime: RuntimeContext, model_name: str) -> tuple[str, int, str]: + """The EPP's KV-event socket on the head node; the topic names the endpoint the events belong to. + + vLLM's topic is ``kv@@``: the producer files the events under + that endpoint's ``address:port`` (the sidecar's port for a proxied decode + worker) and the model the requests name (pkg/kvevents/engineadapter/vllm_adapter.go). + """ + address = self.resolve_worker_host(process.node, runtime.network_interface) + return runtime.head_node_ip, LLM_D_EPP_KV_EVENTS_PORT, f"kv@{address}:{_routed_port(process)}@{model_name}" + + def worker_metrics_port(self, process: Process, runtime: RuntimeContext) -> int | None: + """Every routable worker process serves its own API and metrics, an external-LB DP rank included.""" + return process.http_port if process.http_port > 0 else None + + def worker_endpoint_port(self, process: Process, config: Any, runtime: RuntimeContext) -> int | None: + return process.http_port if process.http_port > 0 else None + def proxied_worker_modes(self, config: Any) -> frozenset[str]: """Prefill/decode: the router reaches each decode worker through its P/D sidecar.""" return frozenset({"decode"}) if _is_pd(config) else frozenset() @@ -243,18 +372,23 @@ def endpoints_document( for process in backend_processes: if process.http_port <= 0: continue - port = process.proxy_port if process.proxy_port is not None else process.http_port endpoints.append( { "name": f"{process.endpoint_mode}-{process.endpoint_index}-{process.node_rank}", "namespace": POOL, "address": self.resolve_worker_host(process.node, network_interface), - "port": str(port), + "port": str(_routed_port(process)), "labels": {ROLE_LABEL: ROLE_BY_MODE[process.endpoint_mode]}, } ) return {"endpoints": endpoints} + def render_url(self, backend_processes: list[Process], network_interface: str | None) -> str: + """A worker's own API (not its proxy), for the token producer: the first prefill worker, else the first.""" + routable = [process for process in backend_processes if process.http_port > 0] + process = next((p for p in routable if p.endpoint_mode == "prefill"), routable[0]) + return f"http://{self.resolve_worker_host(process.node, network_interface)}:{process.http_port}" + def epp_command(self, config: Any, epp_config_path: str) -> list[str]: user_args = self.get_frontend_args_list(config.frontend.args) managed = {str(key).replace("_", "-") for key in (config.frontend.args or {})} & _MANAGED_EPP_FLAGS @@ -291,6 +425,12 @@ def start_frontends( endpoints = self.endpoints_document(backend_processes, runtime.network_interface) (runtime.log_dir / ENDPOINTS_FILE).write_text(yaml.safe_dump(endpoints, sort_keys=False)) epp_document = epp_config_document(config.frontend.epp_config, _container_path(ENDPOINTS_FILE)) + if _plugins(epp_document, PRECISE_PREFIX_PRODUCER) or _plugins(epp_document, TOKEN_PRODUCER): + epp_document = with_kv_events( + epp_document, + self.render_url(backend_processes, runtime.network_interface), + backend.get_served_model_name(runtime.model_path.name), + ) (runtime.log_dir / EPP_CONFIG_FILE).write_text(yaml.safe_dump(epp_document, sort_keys=False)) (runtime.log_dir / ENVOY_CONFIG_FILE).write_text( envoy_config(topology.frontend_port, _container_path(ENVOY_ACCESS_LOG)) diff --git a/src/srtctl/ports.py b/src/srtctl/ports.py index 46c1af916..8790b23f9 100644 --- a/src/srtctl/ports.py +++ b/src/srtctl/ports.py @@ -49,6 +49,9 @@ LLM_D_EPP_HEALTH_PORT = 9003 LLM_D_EPP_METRICS_PORT = 9090 LLM_D_ENVOY_ADMIN_PORT = 9901 +# The EPP's ZMQ subscriber for vLLM KV-cache events (precise prefix-cache routing): it binds +# this port on the router node and every publishing worker connects to it (upstream's 5557). +LLM_D_EPP_KV_EVENTS_PORT = 5557 # First listener of a per-worker proxy a router sends a worker's traffic through # (llm-d's P/D sidecar in front of a decode worker); see PROXY_PORTS. WORKER_PROXY_PORT_BASE = 9600 diff --git a/tests/snapshots/launch/vllm__llm-d-dp-ranks.txt b/tests/snapshots/launch/vllm__llm-d-dp-ranks.txt new file mode 100644 index 000000000..fc0355488 --- /dev/null +++ b/tests/snapshots/launch/vllm__llm-d-dp-ranks.txt @@ -0,0 +1,572 @@ +# examples/vllm/llm-d-dp-ranks.yaml +# exit_code: 0 +# srun calls: 28 + +## agg_0_node-01_dp0 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5200 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20000 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6100 --data-parallel-rank 0 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 0 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6100@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5557"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp1 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp1.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5201 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20050 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6132 --data-parallel-rank 1 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 1 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6132@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5556"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp2 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp2.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5202 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20100 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6164 --data-parallel-rank 2 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 2 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6164@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5555"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp3 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp3.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5203 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20150 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6196 --data-parallel-rank 3 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 3 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6196@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5554"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp4 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp4.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5204 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20200 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6228 --data-parallel-rank 4 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 4 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6228@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5553"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp5 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp5.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5205 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20250 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6260 --data-parallel-rank 5 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 5 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6260@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5552"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp6 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp6.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5206 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20300 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6292 --data-parallel-rank 6 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 6 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6292@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5551"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp7 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp7.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5207 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20350 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6324 --data-parallel-rank 7 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 7 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6324@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5550"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp8 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp8.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5208 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20400 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6100 --data-parallel-rank 8 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 0 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6100@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5549"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp9 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp9.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5209 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20450 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6132 --data-parallel-rank 9 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 1 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6132@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5548"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp10 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp10.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5210 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20500 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6164 --data-parallel-rank 10 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 2 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6164@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5547"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp11 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp11.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5211 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20550 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6196 --data-parallel-rank 11 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 3 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6196@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5546"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp12 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp12.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5212 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20600 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6228 --data-parallel-rank 12 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 4 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6228@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5545"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp13 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp13.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5213 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20650 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6260 --data-parallel-rank 13 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 5 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6260@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5544"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp14 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp14.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5214 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20700 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6292 --data-parallel-rank 14 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 6 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6292@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5543"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp15 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp15.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5215 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20750 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6324 --data-parallel-rank 15 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 7 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6324@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5542"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## llm-d-epp_0 +nodelist: node-01 +output: /logs/node-01_llm-d-epp_0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +command: | + epp --pool-name=srtctl --pool-namespace=srtctl --config-file=/logs/llm-d-epp-config.yaml --grpc-port=9002 --grpc-health-port=9003 --metrics-port=9090 --secure-serving=false + +## llm-d-envoy_0 +nodelist: node-01 +output: /logs/node-01_llm-d-envoy_0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +command: | + envoy -c /logs/llm-d-envoy.yaml --disable-hot-restart + +## service_dcgm-exporter_node-01 +nodelist: node-01 +output: /logs/service_dcgm-exporter_node-01.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-02 +nodelist: node-02 +output: /logs/service_dcgm-exporter_node-02.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-03 +nodelist: node-03 +output: /logs/service_dcgm-exporter_node-03.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-04 +nodelist: node-04 +output: /logs/service_dcgm-exporter_node-04.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_node-exporter_node-01 +nodelist: node-01 +output: /logs/service_node-exporter_node-01.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-02 +nodelist: node-02 +output: /logs/service_node-exporter_node-02.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-03 +nodelist: node-03 +output: /logs/service_node-exporter_node-03.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-04 +nodelist: node-04 +output: /logs/service_node-exporter_node-04.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## tachometer +nodelist: node-01 +output: /logs/tachometer.out +srun_export_env: + POLARS_MAX_THREADS: 4 +use_bash_wrapper: False +command: | + tachometer-scraper --config /logs/tachometer_config.toml --local-dir /logs/tachometer/local --sync-interval 120 + +## +nodelist: node-01 +output: /logs/benchmark.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + SRTCTL_FRONTEND_TYPE: llm-d + SRT_FRONTEND_HOST: 127.0.0.1 + SRT_FRONTEND_PORT: 8000 +command: | + bash /srtctl-benchmarks/sa-bench/bench.sh http://localhost:8000 128 128 16x32 inf /model Qwen/Qwen3-30B-A3B false 16 0 0 0.8 10 2 '' true random '' false diff --git a/tests/test_design_rules.py b/tests/test_design_rules.py index dbcc2fb43..e1a6cab6c 100644 --- a/tests/test_design_rules.py +++ b/tests/test_design_rules.py @@ -152,6 +152,9 @@ class Rule: frozenset( { ("backends/vllm.py", "grpc_port + 1"), + # Not a listener: a KV-event publisher dials the router's one subscriber port, and vLLM + # adds the publisher's DP rank to the port it is given, so the rank is taken off first. + ("backends/vllm.py", "port - rank"), # KVBM_ZMQ_PORTS allocates a two-port block; the ACK port is the block's second port. ("cli/mixins/worker_stage.py", "leader.kvbm_zmq_port + 1"), } diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 3b6650adb..75af058dc 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -30,7 +30,7 @@ def test_interactive_discovers_curated_examples(): assert SGLANG_ROUTER_DISAGG in examples assert MOCKER_EXAMPLE in examples - assert len(TOPOLOGY_EXAMPLES) == 24 + assert len(TOPOLOGY_EXAMPLES) == 25 # ============================================================================= diff --git a/tests/test_llm_d_frontend.py b/tests/test_llm_d_frontend.py index 8c0b5adfe..c18a6f7ea 100644 --- a/tests/test_llm_d_frontend.py +++ b/tests/test_llm_d_frontend.py @@ -6,6 +6,7 @@ from __future__ import annotations import copy +import json from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -26,8 +27,15 @@ EPP_CONFIG_FILE, epp_config_document, parse_ready_endpoints, + with_kv_events, +) +from srtctl.ports import ( + LLM_D_ENVOY_ADMIN_PORT, + LLM_D_EPP_GRPC_PORT, + LLM_D_EPP_KV_EVENTS_PORT, + LLM_D_EPP_METRICS_PORT, + WORKER_PROXY_PORT_BASE, ) -from srtctl.ports import LLM_D_ENVOY_ADMIN_PORT, LLM_D_EPP_GRPC_PORT, LLM_D_EPP_METRICS_PORT, WORKER_PROXY_PORT_BASE from srtctl.services.implicit import effective_services from srtctl.services.registry import ServiceLaunchContext, get_service_kind from tests.launch_snapshots import EXAMPLES_DIR, render_launch_plan @@ -316,3 +324,175 @@ def test_engine_must_be_vllm() -> None: recipe["roles"]["agg"]["args"] = {"served-model-name": "m"} with pytest.raises(ValidationError, match="requires backend"): _load(recipe) + + +DP_RANKS = EXAMPLES_DIR / "vllm/llm-d-dp-ranks.yaml" + + +def _precise(recipe: dict) -> dict: + """The disaggregated example with precise prefix-cache routing on its prefill worker.""" + epp = recipe["frontend"]["epp_config"] + epp["plugins"] = [ + {"type": "token-producer"}, + {"type": "precise-prefix-cache-producer"}, + *(plugin for plugin in epp["plugins"] if plugin["type"] != "approx-prefix-cache-producer"), + ] + recipe["roles"]["prefill"]["kv_events"] = True + return recipe + + +def test_external_lb_ranks_are_endpoints_with_their_own_ports() -> None: + """data-parallel-external-lb: one `vllm serve` per DP rank, each an endpoint on its own port.""" + config = load_config(DP_RANKS) + endpoints = [Endpoint("agg", 0, ("node0", "node1"), frozenset(range(8)))] + processes = config.worker_processes(endpoints) + assert [p.dp_rank for p in processes] == list(range(16)) + assert all(p.http_port > 0 for p in processes) + assert len({(p.node, p.http_port) for p in processes}) == 16 + assert processes[9].engine_suffix == "_dp9" + with patch("srtctl.frontends.static_router.get_hostname_ip", side_effect=lambda node, _: node): + document = LLMDFrontend().endpoints_document(processes, None) + assert len(document["endpoints"]) == 16 + frontend = LLMDFrontend() + assert frontend.health_expectations(config, processes) == (0, 16, "16 llm-d endpoints") + assert frontend.worker_metrics_port(processes[9], MagicMock()) == processes[9].http_port + + +def test_external_lb_ranks_publish_kv_events_to_the_epp() -> None: + """Every rank connects to the EPP's one socket (vLLM adds the rank back) under its own endpoint's topic.""" + plan = render_launch_plan(DP_RANKS) + assert "# exit_code: 0" in plan + steps = [line[3:] for line in plan.splitlines() if line.startswith("## ")] + assert len(steps) == len(set(steps)) + assert "agg_0_node-02_dp9" in steps + rank9 = plan.split("## agg_0_node-02_dp9")[1].split("## ")[0] + assert "--data-parallel-rank 9 --data-parallel-address" in rank9 + assert "--data-parallel-external-lb" in rank9 and "--headless" not in rank9 + kv_events = json.loads(rank9.split("--kv-events-config '")[1].split("'")[0]) + port = rank9.split("--port ")[1].split()[0] + assert kv_events["endpoint"] == f"tcp://127.0.0.1:{LLM_D_EPP_KV_EVENTS_PORT - 9}" + assert kv_events["topic"] == f"kv@127.0.0.1:{port}@Qwen/Qwen3-30B-A3B" + assert kv_events["publisher"] == "zmq" and kv_events["enable_kv_cache_events"] is True + + +def test_external_lb_decode_ranks_each_get_a_sidecar() -> None: + recipe = _precise(_recipe()) + for role in ("prefill", "decode"): + recipe["roles"][role]["gpus"] = 2 + recipe["roles"][role]["args"].update({"data-parallel-size": 2, "data-parallel-external-lb": True}) + config = _load(recipe) + endpoints = [ + Endpoint("prefill", 0, ("node0",), frozenset({0, 1})), + Endpoint("decode", 0, ("node1",), frozenset({0, 1})), + ] + processes = config.worker_processes(endpoints) + decode = [p for p in processes if p.endpoint_mode == "decode"] + assert [p.proxy_port for p in decode] == [WORKER_PROXY_PORT_BASE, WORKER_PROXY_PORT_BASE + 1] + kind = get_service_kind("llm-d-sidecar") + assert [kind.attaches_to(p) for p in processes] == [False, False, True, True] + # The decode topic names the sidecar, the endpoint the EPP routes to. + runtime = SimpleNamespace(network_interface=None, head_node_ip="10.0.0.9") + with patch("srtctl.frontends.static_router.get_hostname_ip", return_value="10.0.0.2"): + assert LLMDFrontend().kv_events_subscriber(decode[1], runtime, "m") == ( + "10.0.0.9", + LLM_D_EPP_KV_EVENTS_PORT, + f"kv@10.0.0.2:{WORKER_PROXY_PORT_BASE + 1}@m", + ) + + +def test_kv_events_need_the_precise_producer_and_vice_versa() -> None: + recipe = _recipe() + recipe["roles"]["prefill"]["kv_events"] = True + with pytest.raises(ValidationError, match="add a precise-prefix-cache-producer plugin"): + _load(recipe) + recipe = _precise(_recipe()) + del recipe["roles"]["prefill"]["kv_events"] + with pytest.raises(ValidationError, match="set roles..kv_events"): + _load(recipe) + + +@pytest.mark.parametrize( + ("edit", "message"), + [ + ( + lambda r: r["frontend"]["epp_config"]["plugins"][1].update( + {"parameters": {"kvEventsConfig": {"zmqEndpoint": "tcp://*:1"}}} + ), + "kvEventsConfig.zmqEndpoint, which srtctl manages", + ), + (lambda r: r["frontend"]["epp_config"]["plugins"].pop(0), "add a token-producer plugin"), + ( + lambda r: r["frontend"]["epp_config"]["plugins"][0].update({"parameters": {"vllm": {"url": "http://x"}}}), + "sets token-producer vllm.url", + ), + ( + lambda r: r["frontend"].update({"placement": {"node": "first_decode"}}), + "publish to the EPP on the head node", + ), + ( + lambda r: r["roles"]["prefill"].update( + {"gpus": 2, "args": {**r["roles"]["prefill"]["args"], "data-parallel-size": 2}} + ), + "set roles.prefill.args.data-parallel-external-lb: true", + ), + ], +) +def test_precise_prefix_routing_rules(edit, message: str) -> None: + recipe = _precise(_recipe()) + edit(recipe) + with pytest.raises(ValidationError, match=message): + _load(recipe) + + +def test_with_kv_events_wires_the_socket_and_the_render_url() -> None: + document = { + "plugins": [ + {"type": "token-producer"}, + {"type": "token-producer", "name": "approx", "parameters": {"estimate": {}}}, + {"type": "precise-prefix-cache-producer", "parameters": {"kvEventsConfig": {"concurrency": 8}}}, + {"type": "queue-scorer"}, + ] + } + with_kv_events(document, "http://10.0.0.1:6100", "m") + token, estimate, producer, scorer = document["plugins"] + assert token["parameters"] == {"modelName": "m", "vllm": {"url": "http://10.0.0.1:6100"}} + assert estimate["parameters"] == {"estimate": {}} + assert producer["parameters"]["kvEventsConfig"] == { + "concurrency": 8, + "zmqEndpoint": f"tcp://*:{LLM_D_EPP_KV_EVENTS_PORT}", + "discoverPods": False, + } + assert scorer == {"type": "queue-scorer"} + + +def test_start_frontends_points_the_token_producer_at_the_prefill_worker(tmp_path: Path) -> None: + config = _load(_precise(_recipe())) + runtime = SimpleNamespace( + network_interface=None, + log_dir=tmp_path, + container_image="model.sqsh", + container_mounts={}, + environment={}, + srun_options={}, + model_path=Path("/models/qwen"), + nodes=SimpleNamespace(het_group_for=lambda node: None), + ) + processes = [ + Process("node1", frozenset({0}), 7501, 6100, "decode", 0, proxy_port=9600), + Process("node0", frozenset({0}), 7500, 6101, "prefill", 0), + ] + ips = {"node0": "10.0.0.1", "node1": "10.0.0.2"} + with ( + patch("srtctl.frontends.static_router.get_hostname_ip", side_effect=lambda node, _: ips[node]), + patch.object(LLMDFrontend, "wait_for_workers"), + patch.object(LLMDFrontend, "start_process", return_value=MagicMock()), + ): + LLMDFrontend().start_frontends( + SimpleNamespace(frontend_nodes=["node0"], frontend_port=8000), runtime, config, config.backend, processes + ) + plugins = {p["type"]: p for p in yaml.safe_load((tmp_path / EPP_CONFIG_FILE).read_text())["plugins"]} + assert plugins["token-producer"]["parameters"] == { + "modelName": "Qwen/Qwen3-0.6B", + "vllm": {"url": "http://10.0.0.1:6101"}, + } + assert plugins["precise-prefix-cache-producer"]["parameters"]["kvEventsConfig"]["discoverPods"] is False From 1c3080ec2de96377610e8a383b6ffe2adab95597 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 6 Oct 2026 18:31:08 -0500 Subject: [PATCH 05/25] feat(llm-d): run the no-Kubernetes guide's scorers without an epp_config The EPP falls back to its own default only for a configuration file without plugins, and srtctl's always carries file discovery, so a recipe without frontend.epp_config ran no scorers. srtctl now writes the scorers of llm-d's no-Kubernetes guide, split into prefill and decode profiles for a prefill/decode job. Signed-off-by: Cam Quilici --- docs/llm-d.md | 13 +++++++--- docs/schema-reference.md | 2 +- docs/schema/recipe.schema.json | 2 +- src/srtctl/core/schema.py | 6 ++--- src/srtctl/frontends/llm_d.py | 45 ++++++++++++++++++++++++++++++-- tests/test_llm_d_frontend.py | 47 ++++++++++++++++++++++++++++++++-- 6 files changed, 103 insertions(+), 12 deletions(-) diff --git a/docs/llm-d.md b/docs/llm-d.md index 38a15e400..1002739af 100644 --- a/docs/llm-d.md +++ b/docs/llm-d.md @@ -140,9 +140,16 @@ engine: ``` `frontend.args` are passed to the EPP; the pool, configuration, and port flags are -srtctl's. Prefill/decode needs `epp_config` with `prefill` and `decode` scheduling -profiles and the `disagg-profile-handler`; an aggregate job without `epp_config` runs -the EPP's built-in default profile. See +srtctl's. A prefill/decode `epp_config` needs `prefill` and `decode` scheduling profiles +and the `disagg-profile-handler`. Without `epp_config`, srtctl runs the scorers of llm-d's +no-Kubernetes guide +([`config.yaml`](https://github.com/llm-d/llm-d/blob/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment/router/epp/config.yaml)): +queue depth (2), KV-cache utilization (2), approximate prefix cache (3), and +no-hit LRU (2). A prefill/decode job gets them as a `prefill` profile, a `decode` +profile that scores by queue depth and KV-cache utilization alone, and the +`disagg-profile-handler` with `always-disagg-pd-decider`. (The EPP's own default applies +only to a configuration file without plugins, and srtctl's always carries file +discovery.) See [`examples/vllm/llm-d-disagg.yaml`](https://github.com/NVIDIA/srt-slurm/blob/main/examples/vllm/llm-d-disagg.yaml) and [`examples/vllm/llm-d-agg.yaml`](https://github.com/NVIDIA/srt-slurm/blob/main/examples/vllm/llm-d-agg.yaml). diff --git a/docs/schema-reference.md b/docs/schema-reference.md index 910aa3117..cc926a355 100644 --- a/docs/schema-reference.md +++ b/docs/schema-reference.md @@ -130,7 +130,7 @@ Frontend/router configuration. | `nginx_session_affinity_header` | str | `'X-Dynamo-Session-ID'` | Header hashed when affinity is on (default ``X-Dynamo-Session-ID``). Set ``X-Correlation-ID`` for clients (e.g. aiperf) that carry the session id in that header instead. | | `nginx_keepalive_timeout` | str | `'600s'` | Idle timeout for client and upstream keepalive connections in the generated nginx.conf (default "600s"). nginx's own default is 75s, which closes a session's connection during the long recorded think-time of an agentic replay; the client's next write on that pooled socket then fails with "broken pipe" / "server disconnected" and nothing is logged server-side. | | `worker_selection` | dict[str, Any] \| None | `None` | Inline Dynamo worker-selection policy configuration. srtctl writes this mapping under the top-level ``worker_selection`` key in a generated router policy YAML and passes it to the Dynamo frontend via ``--router-policy-config``. | -| `epp_config` | dict[str, Any] \| None | `None` | llm-d Endpoint Picker configuration (``EndpointPickerConfig``: ``plugins``, ``schedulingProfiles``, ...) for ``frontend.type: llm-d``. srtctl writes it to a file with the ``file-discovery`` plugin and ``dataLayer.discovery`` added, so the EPP reads the job's workers from the endpoints file srtctl renders. Required for prefill/decode jobs (the scheduler's P/D profiles); omitted, an aggregate job runs the EPP's built-in default profile. | +| `epp_config` | dict[str, Any] \| None | `None` | llm-d Endpoint Picker configuration (``EndpointPickerConfig``: ``plugins``, ``schedulingProfiles``, ...) for ``frontend.type: llm-d``. srtctl writes it to a file with the ``file-discovery`` plugin and ``dataLayer.discovery`` added, so the EPP reads the job's workers from the endpoints file srtctl renders. Omitted, srtctl runs the scorers of llm-d's no-Kubernetes guide (in prefill and decode profiles for a prefill/decode job). | | `args` | dict[str, Any] \| None | `None` | CLI arguments passed to the frontend/router process | | `env` | dict[str, str] \| None | `None` | Environment variables for frontend processes | | `container_image` | str \| None | `None` | Optional router-specific image. Static routers use the model/backend image when omitted. | diff --git a/docs/schema/recipe.schema.json b/docs/schema/recipe.schema.json index d3288f824..e6cd20cb8 100644 --- a/docs/schema/recipe.schema.json +++ b/docs/schema/recipe.schema.json @@ -927,7 +927,7 @@ "type": "null" } ], - "description": "llm-d Endpoint Picker configuration (``EndpointPickerConfig``: ``plugins``, ``schedulingProfiles``, ...) for ``frontend.type: llm-d``. srtctl writes it to a file with the ``file-discovery`` plugin and ``dataLayer.discovery`` added, so the EPP reads the job's workers from the endpoints file srtctl renders. Required for prefill/decode jobs (the scheduler's P/D profiles); omitted, an aggregate job runs the EPP's built-in default profile.", + "description": "llm-d Endpoint Picker configuration (``EndpointPickerConfig``: ``plugins``, ``schedulingProfiles``, ...) for ``frontend.type: llm-d``. srtctl writes it to a file with the ``file-discovery`` plugin and ``dataLayer.discovery`` added, so the EPP reads the job's workers from the endpoints file srtctl renders. Omitted, srtctl runs the scorers of llm-d's no-Kubernetes guide (in prefill and decode profiles for a prefill/decode job).", "default": null }, "args": { diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index abc31c7fa..236c48a4c 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -2569,9 +2569,9 @@ class FrontendConfig: ``plugins``, ``schedulingProfiles``, ...) for ``frontend.type: llm-d``. srtctl writes it to a file with the ``file-discovery`` plugin and ``dataLayer.discovery`` added, so the EPP reads the job's workers from - the endpoints file srtctl renders. Required for prefill/decode jobs - (the scheduler's P/D profiles); omitted, an aggregate job runs the - EPP's built-in default profile. + the endpoints file srtctl renders. Omitted, srtctl runs the scorers of + llm-d's no-Kubernetes guide (in prefill and decode profiles for a + prefill/decode job). args: CLI arguments passed to the frontend/router process env: Environment variables for frontend processes container_image: Optional router-specific image. Static routers use the diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py index 10649a19f..02246fbd6 100644 --- a/src/srtctl/frontends/llm_d.py +++ b/src/srtctl/frontends/llm_d.py @@ -105,6 +105,44 @@ def _container_path(name: str) -> str: return str(Path("/logs") / name) +# Scorers and weights of llm-d's no-Kubernetes guide +# (guides/no-kubernetes-deployment/router/epp/config.yaml), the scheduler srtctl runs when a +# recipe gives no frontend.epp_config. A prefill/decode job scores prefill workers with all +# of them and decode workers by load alone. +_DEFAULT_SCORERS = { + "queue-scorer": 2, + "kv-cache-utilization-scorer": 2, + "prefix-cache-scorer": 3, + "no-hit-lru-scorer": 2, +} +_DEFAULT_DECODE_SCORERS = ("queue-scorer", "kv-cache-utilization-scorer") + + +def default_epp_config(pd: bool) -> dict[str, Any]: + """The scheduler for a recipe without ``frontend.epp_config``: the guide's scorers, split into P/D profiles for ``pd``.""" + + def profile(name: str, scorers: Any, role_filter: str | None = None) -> dict[str, Any]: + refs = [{"pluginRef": scorer, "weight": _DEFAULT_SCORERS[scorer]} for scorer in scorers] + return {"name": name, "plugins": [{"pluginRef": role_filter}, *refs] if role_filter else refs} + + plugins: list[dict[str, Any]] = [{"type": scorer} for scorer in _DEFAULT_SCORERS] + if not pd: + return {"plugins": plugins, "schedulingProfiles": [profile("default", _DEFAULT_SCORERS)]} + plugins += [ + {"type": "prefill-filter"}, + {"type": "decode-filter"}, + {"type": "always-disagg-pd-decider"}, + {"type": "disagg-profile-handler", "parameters": {"deciders": {"prefill": "always-disagg-pd-decider"}}}, + ] + return { + "plugins": plugins, + "schedulingProfiles": [ + profile("prefill", _DEFAULT_SCORERS, "prefill-filter"), + profile("decode", _DEFAULT_DECODE_SCORERS, "decode-filter"), + ], + } + + def epp_config_document(epp_config: dict[str, Any] | None, endpoints_path: str) -> dict[str, Any]: """The recipe's ``EndpointPickerConfig`` with srtctl's file discovery added. @@ -229,7 +267,7 @@ def validate(self, config: Any) -> None: ) self._validate_kv_events(config) if _is_pd(config): - if not epp_config.get("schedulingProfiles"): + if frontend.epp_config is not None and not epp_config.get("schedulingProfiles"): raise ValueError( "frontend.type: llm-d with prefill and decode workers needs frontend.epp_config with " "prefill and decode schedulingProfiles and a disaggregation profile handler" @@ -424,7 +462,10 @@ def start_frontends( endpoints = self.endpoints_document(backend_processes, runtime.network_interface) (runtime.log_dir / ENDPOINTS_FILE).write_text(yaml.safe_dump(endpoints, sort_keys=False)) - epp_document = epp_config_document(config.frontend.epp_config, _container_path(ENDPOINTS_FILE)) + epp_config = config.frontend.epp_config + epp_document = epp_config_document( + default_epp_config(_is_pd(config)) if epp_config is None else epp_config, _container_path(ENDPOINTS_FILE) + ) if _plugins(epp_document, PRECISE_PREFIX_PRODUCER) or _plugins(epp_document, TOKEN_PRODUCER): epp_document = with_kv_events( epp_document, diff --git a/tests/test_llm_d_frontend.py b/tests/test_llm_d_frontend.py index c18a6f7ea..3aeedea4e 100644 --- a/tests/test_llm_d_frontend.py +++ b/tests/test_llm_d_frontend.py @@ -149,13 +149,56 @@ def test_declared_sidecar_must_sit_on_proxied_workers() -> None: _load(recipe) -def test_pd_requires_an_epp_scheduler() -> None: +def test_pd_epp_config_needs_scheduling_profiles() -> None: recipe = _recipe() - del recipe["frontend"]["epp_config"] + del recipe["frontend"]["epp_config"]["schedulingProfiles"] with pytest.raises(ValidationError, match="needs frontend.epp_config"): _load(recipe) +@pytest.mark.parametrize("path", [AGG, DISAGG]) +def test_without_epp_config_srtctl_runs_the_guide_scorers(path: Path, tmp_path: Path) -> None: + """The EPP's own default only applies to a plugin-less file; srtctl's always carries discovery.""" + recipe = _recipe(path) + del recipe["frontend"]["epp_config"] + config = _load(recipe) + runtime = SimpleNamespace( + network_interface=None, + log_dir=tmp_path, + container_image="model.sqsh", + container_mounts={}, + environment={}, + srun_options={}, + nodes=SimpleNamespace(het_group_for=lambda node: None), + ) + processes = [Process("node0", frozenset({0}), 7500, 6100, "agg", 0)] + with ( + patch("srtctl.frontends.static_router.get_hostname_ip", return_value="10.0.0.1"), + patch.object(LLMDFrontend, "wait_for_workers"), + patch.object(LLMDFrontend, "start_process", return_value=MagicMock()), + ): + LLMDFrontend().start_frontends( + SimpleNamespace(frontend_nodes=["node0"], frontend_port=8000), runtime, config, config.backend, processes + ) + epp = yaml.safe_load((tmp_path / EPP_CONFIG_FILE).read_text()) + profiles = {profile["name"]: profile["plugins"] for profile in epp["schedulingProfiles"]} + guide = [ + {"pluginRef": "queue-scorer", "weight": 2}, + {"pluginRef": "kv-cache-utilization-scorer", "weight": 2}, + {"pluginRef": "prefix-cache-scorer", "weight": 3}, + {"pluginRef": "no-hit-lru-scorer", "weight": 2}, + ] + if path == AGG: + assert profiles == {"default": guide} + else: + assert profiles == { + "prefill": [{"pluginRef": "prefill-filter"}, *guide], + "decode": [{"pluginRef": "decode-filter"}, *guide[:2]], + } + assert {"type": "always-disagg-pd-decider"} in epp["plugins"] + assert epp["plugins"][-1]["type"] == "file-discovery" + + def test_recipe_cannot_configure_discovery() -> None: recipe = _recipe() recipe["frontend"]["epp_config"]["dataLayer"] = {"discovery": {"pluginRef": "mine"}} From d04b5d2d0c467e07025157b33528fe184fde55dd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 6 Oct 2026 18:47:06 -0500 Subject: [PATCH 06/25] feat(llm-d): run the P/D sidecar for NIXL inside a MultiConnector A decode worker that runs NIXL next to vLLM's OffloadingConnector in a MultiConnector (llm-d's tiered wide-EP layout) now gets the nixlv2 sidecar protocol instead of being rejected. Signed-off-by: Cam Quilici --- docs/llm-d.md | 6 ++++-- src/srtctl/backends/vllm.py | 31 ++++++++++++++++------------ src/srtctl/services/llm_d_sidecar.py | 20 +++++++++++------- tests/test_llm_d_frontend.py | 24 +++++++++++++++++++-- 4 files changed, 56 insertions(+), 25 deletions(-) diff --git a/docs/llm-d.md b/docs/llm-d.md index 1002739af..1531e5d84 100644 --- a/docs/llm-d.md +++ b/docs/llm-d.md @@ -62,14 +62,16 @@ srtctl sets `--port` (the worker's `Process.proxy_port`), `--model-server-port` The allocator hands out the proxy port (`PROXY_PORTS`) for every routable worker of a mode the frontend proxies (`Frontend.proxied_worker_modes`), so two decode workers on one node never collide. The `--kv-connector` protocol comes from the decode workers' vLLM -connector class (`VLLMBackend.kv_connector_class`, which reads a role's own +connector class (`VLLMBackend.kv_connector_classes`, which reads a role's own `kv-transfer-config` before `connector`): | Decode connector | Sidecar protocol | | --- | --- | | `NixlConnector` | `nixlv2` | -Any other decode connector is rejected at load time. +A `MultiConnector` maps through the first connector it wraps that has a protocol, so +NIXL next to vLLM's `OffloadingConnector` (CPU KV offloading, as in llm-d's tiered +wide-EP guide) runs `nixlv2`. Any other decode connector is rejected at load time. ## Endpoint discovery diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index 6874926c9..addbd3d2d 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -437,12 +437,13 @@ def kv_connector_for_mode(self, mode: WorkerMode) -> KVConnector | None: """The connector table row for a mode; None for no connector or a raw JSON ``--kv-transfer-config``.""" return kv_connector_row(self.connector_for_mode(mode)) - def kv_connector_class(self, mode: WorkerMode) -> str | None: - """The vLLM connector class a mode's workers run: ``kv_connector`` of their ``--kv-transfer-config``. + def kv_connector_classes(self, mode: WorkerMode) -> tuple[str, ...]: + """The vLLM connector classes a mode's workers run: ``kv_connector`` of their ``--kv-transfer-config``. - A ``kv-transfer-config`` written in the role's args wins, as it does on the - command line; otherwise the mode's connector (a table row or raw JSON). - None when the mode runs no connector. + A ``MultiConnector`` is followed by the classes of the connectors it wraps + (``kv_connector_extra_config.connectors``). A ``kv-transfer-config`` written + in the role's args wins, as it does on the command line; otherwise the mode's + connector (a table row or raw JSON). Empty when the mode runs no connector. """ explicit = next( ( @@ -452,16 +453,20 @@ def kv_connector_class(self, mode: WorkerMode) -> str | None: ), None, ) - if explicit is not None: - payload = explicit if isinstance(explicit, Mapping) else json.loads(str(explicit)) - return payload.get("kv_connector") row = self.kv_connector_for_mode(mode) - if row is not None: - return row.kv_connector connector = self.connector_for_mode(mode) - if not connector or connector.lower() in ("null", "none"): - return None - return json.loads(connector).get("kv_connector") + if explicit is not None: + payload = explicit if isinstance(explicit, Mapping) else json.loads(str(explicit)) + elif row is not None: + return (row.kv_connector,) + elif not connector or connector.lower() in ("null", "none"): + return () + else: + payload = json.loads(connector) + wrapped = (payload.get("kv_connector_extra_config") or {}).get("connectors") or [] + return tuple( + name for name in (payload.get("kv_connector"), *(inner.get("kv_connector") for inner in wrapped)) if name + ) def discovers_workers(self) -> bool: """Whether the prefill/decode workers register with the router over its discovery endpoint. diff --git a/src/srtctl/services/llm_d_sidecar.py b/src/srtctl/services/llm_d_sidecar.py index 8a354e662..a70219b8e 100644 --- a/src/srtctl/services/llm_d_sidecar.py +++ b/src/srtctl/services/llm_d_sidecar.py @@ -45,19 +45,23 @@ def sidecar_kv_connector(backend: Any) -> str: - """The ``--kv-connector`` protocol for the decode workers' KV connector; raises ``ValueError`` when unmapped.""" + """The ``--kv-connector`` protocol for the decode workers' KV connector; raises ``ValueError`` when unmapped. + + A ``MultiConnector`` (P/D transfer next to KV offloading) maps through the first + connector it wraps that has a protocol. + """ from srtctl.backends.vllm import VLLMBackend # The sidecar drives vLLM's KV transfer protocol; another engine's decode has no row. - connector = backend.kv_connector_class("decode") if isinstance(backend, VLLMBackend) else None - try: - return SIDECAR_KV_CONNECTORS[str(connector)] - except KeyError: + connectors = backend.kv_connector_classes("decode") if isinstance(backend, VLLMBackend) else () + protocol = next((SIDECAR_KV_CONNECTORS[name] for name in connectors if name in SIDECAR_KV_CONNECTORS), None) + if protocol is None: supported = ", ".join(f"{name} ({protocol})" for name, protocol in SIDECAR_KV_CONNECTORS.items()) raise ValueError( - f"the llm-d P/D sidecar has no protocol for the {backend.type} decode KV connector {connector!r}; " - f"supported (vLLM): {supported}" - ) from None + f"the llm-d P/D sidecar has no protocol for the {backend.type} decode KV connector " + f"{' > '.join(connectors) or None!r}; supported (vLLM, alone or in a MultiConnector): {supported}" + ) + return protocol @register_service(LLM_D_SIDECAR_TYPE) diff --git a/tests/test_llm_d_frontend.py b/tests/test_llm_d_frontend.py index 3aeedea4e..f6d1b51e0 100644 --- a/tests/test_llm_d_frontend.py +++ b/tests/test_llm_d_frontend.py @@ -37,6 +37,7 @@ WORKER_PROXY_PORT_BASE, ) from srtctl.services.implicit import effective_services +from srtctl.services.llm_d_sidecar import sidecar_kv_connector from srtctl.services.registry import ServiceLaunchContext, get_service_kind from tests.launch_snapshots import EXAMPLES_DIR, render_launch_plan @@ -220,8 +221,27 @@ def test_explicit_kv_transfer_config_selects_the_protocol() -> None: recipe["roles"]["decode"]["args"]["kv-transfer-config"] = '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' config = _load(recipe) assert isinstance(config.backend, VLLMBackend) - assert config.backend.kv_connector_class("decode") == "NixlConnector" - assert config.backend.kv_connector_class("prefill") is None + assert config.backend.kv_connector_classes("decode") == ("NixlConnector",) + assert config.backend.kv_connector_classes("prefill") == () + + +def test_multi_connector_maps_through_its_transfer_connector() -> None: + """NIXL next to CPU offloading (llm-d's tiered wide-EP guide) still speaks nixlv2.""" + recipe = _recipe() + recipe["engine"]["connector"] = None + recipe["roles"]["decode"]["args"]["kv-transfer-config"] = { + "kv_connector": "MultiConnector", + "kv_role": "kv_both", + "kv_connector_extra_config": { + "connectors": [ + {"kv_connector": "OffloadingConnector", "kv_role": "kv_both"}, + {"kv_connector": "NixlConnector", "kv_role": "kv_both"}, + ] + }, + } + config = _load(recipe) + assert config.backend.kv_connector_classes("decode") == ("MultiConnector", "OffloadingConnector", "NixlConnector") + assert sidecar_kv_connector(config.backend) == "nixlv2" def test_one_router_replica() -> None: From 1ac5bc659dd7eae189a1c423679e577d6e63f3de Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 6 Oct 2026 19:37:53 -0500 Subject: [PATCH 07/25] docs(llm-d): describe DP-rank and KV-event routing and offloaded tiers Signed-off-by: Cam Quilici --- docs/llm-d.md | 5 +++++ src/srtctl/frontends/llm_d.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/docs/llm-d.md b/docs/llm-d.md index 1531e5d84..e9cb08884 100644 --- a/docs/llm-d.md +++ b/docs/llm-d.md @@ -226,6 +226,11 @@ srtctl wires the rest: - The token producer's `vllm.url` is the first prefill worker's (else the first worker's) own vLLM API, and `modelName` defaults to the served model name. +With vLLM's `OffloadingConnector` and its `self_describing_kv_events: true`, a worker also +reports the blocks it offloads to host memory; the producer indexes them as the `cpu` tier +and weights tiers by `indexerConfig.kvCacheBackendConfigs` (`gpu` 1.0, `cpu` 0.8 by +default). + The roles that publish and the producer go together: either without the other is rejected, as are a producer `kvEventsConfig` that sets the socket or discovery, a `token-producer` `vllm.url`, and a router not on the head node. A role with diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py index 02246fbd6..ddbfe9274 100644 --- a/src/srtctl/frontends/llm_d.py +++ b/src/srtctl/frontends/llm_d.py @@ -20,6 +20,11 @@ Envoy's admin ``/ready`` answers and the EPP reports every endpoint ready (``llm_d_epp_ready_endpoints``: endpoints whose metrics it scrapes). +An external-LB DP rank (``data-parallel-external-lb``) is an endpoint of its own. +With ``roles..kv_events`` the workers publish their KV-cache events to the +EPP's precise prefix-cache producer on one socket srtctl owns +(``kv_events_subscriber``, ``with_kv_events``). + Upstream, pinned: llm-d-router v0.11.0 (https://github.com/llm-d/llm-d-router/tree/a5cbe600ebade00cf3e9885beaf2bfacddeabce1): ``cmd/epp/runner/runner.go`` (``runWithFileDiscovery``), From 35234a6101495fc1e86b027649398201f3e1bd24 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 7 Oct 2026 17:17:47 -0500 Subject: [PATCH 08/25] fix(llm-d): type-check the sidecar's default command after the upstream sync Signed-off-by: Cam Quilici --- src/srtctl/services/llm_d_sidecar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/srtctl/services/llm_d_sidecar.py b/src/srtctl/services/llm_d_sidecar.py index a70219b8e..e6edff139 100644 --- a/src/srtctl/services/llm_d_sidecar.py +++ b/src/srtctl/services/llm_d_sidecar.py @@ -98,7 +98,7 @@ def attaches_to(self, process: Process) -> bool: return process.proxy_port is not None def build_command(self, service: ServiceConfig, ctx: ServiceLaunchContext) -> list[str]: - command = list(service.command) if service.command is not None else list(self.default_command) + command = list(service.command) if service.command is not None else list(self.default_command or ()) if ctx.process is None or ctx.config is None or ctx.process.proxy_port is None: managed = [*_PREVIEW_PORTS, "--kv-connector="] else: From 9d5cb476f9997cf6ab61e24fb1e0d4d4b4730207 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 10:46:05 -0500 Subject: [PATCH 09/25] fix(llm-d): reject unsupported nginx readiness topology --- docs/llm-d.md | 7 +++++-- src/srtctl/frontends/llm_d.py | 11 ++++------- tests/test_llm_d_frontend.py | 13 +++++++++++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/llm-d.md b/docs/llm-d.md index e9cb08884..c71728210 100644 --- a/docs/llm-d.md +++ b/docs/llm-d.md @@ -242,8 +242,11 @@ that rank. - vLLM workers over HTTP only; gRPC roles are rejected. - One router replica: the EPP keeps its scheduling state (the approximate prefix index, - in-flight load) in memory, so `enable_multiple_frontends` with more than one router is - rejected. + in-flight load) in memory. Multi-node jobs require `enable_multiple_frontends: false`: + nginx would put the router on a different node from the public endpoint, where readiness + checks expect Envoy's admin and the EPP's metrics listeners. Setting + `num_additional_frontends: 0` still enables nginx and is rejected too. Single-node jobs + never launch nginx, regardless of `enable_multiple_frontends`. - The endpoints file is written once (`watchFile: false`) and the EPP does not eject a worker that fails; a crashed worker or sidecar fails the job instead. - vLLM's `data-parallel-multi-port-external-lb` (one supervisor per node serving a port diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py index ddbfe9274..afea9b3db 100644 --- a/src/srtctl/frontends/llm_d.py +++ b/src/srtctl/frontends/llm_d.py @@ -244,14 +244,11 @@ class LLMDFrontend(StaticRouterFrontend): def validate(self, config: Any) -> None: frontend = config.frontend - if ( - frontend.enable_multiple_frontends - and frontend.num_additional_frontends > 0 - and config.engine_node_count > 1 - ): + if frontend.enable_multiple_frontends and config.engine_node_count > 1: raise ValueError( - "frontend.type: llm-d runs one Endpoint Picker, which keeps its routing state in memory; " - "set frontend.enable_multiple_frontends: false (or num_additional_frontends: 0)" + "frontend.type: llm-d requires Envoy and the Endpoint Picker on the public endpoint's node " + "for readiness checks; multi-node nginx routing is unsupported, even with " + "num_additional_frontends: 0. Set frontend.enable_multiple_frontends: false" ) topology = config.topology for mode, count in ( diff --git a/tests/test_llm_d_frontend.py b/tests/test_llm_d_frontend.py index f6d1b51e0..6d76a23c8 100644 --- a/tests/test_llm_d_frontend.py +++ b/tests/test_llm_d_frontend.py @@ -244,14 +244,23 @@ def test_multi_connector_maps_through_its_transfer_connector() -> None: assert sidecar_kv_connector(config.backend) == "nixlv2" -def test_one_router_replica() -> None: +@pytest.mark.parametrize("additional_frontends", [0, 1, 9]) +def test_multinode_nginx_is_rejected_even_with_one_router(additional_frontends: int) -> None: recipe = _recipe() recipe["frontend"]["enable_multiple_frontends"] = True + recipe["frontend"]["num_additional_frontends"] = additional_frontends recipe["roles"]["decode"]["nodes"] = 1 - with pytest.raises(ValidationError, match="runs one Endpoint Picker"): + with pytest.raises(ValidationError, match="Set frontend.enable_multiple_frontends: false"): _load(recipe) +def test_single_node_allows_multiple_frontends_setting_without_nginx() -> None: + recipe = _recipe(AGG) + recipe["frontend"]["enable_multiple_frontends"] = True + config = _load(recipe) + assert config.engine_node_count == 1 + + def test_epp_config_gets_srtctl_discovery() -> None: user = {"plugins": [{"type": "queue-scorer"}], "dataLayer": {"sources": [{"pluginRef": "m"}]}} original = copy.deepcopy(user) From bd53c28b97500a133268a870e41cab023bd23ed9 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 10:47:29 -0500 Subject: [PATCH 10/25] fix(vllm): include context parallelism in DP rank placement --- src/srtctl/backends/vllm.py | 68 ++------------------------ tests/test_vllm_dp_parallelism.py | 79 +++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 63 deletions(-) create mode 100644 tests/test_vllm_dp_parallelism.py diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index addbd3d2d..ea93af832 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -775,66 +775,10 @@ def _validate_endpoint_parallelism(self, endpoint: Endpoint) -> tuple[int, int]: ) return dp_size, model_parallel_size - def _get_int_flag(self, mode: WorkerMode, name: str, default: int = 1) -> int: - """Read a positive integer CLI flag from the mode config.""" - config = self.get_config_for_mode(mode) - raw = config.get(name) - if raw is None: - raw = config.get(name.replace("-", "_")) - if raw is None: - return default - try: - value = int(raw) - except (TypeError, ValueError) as exc: - raise ValueError(f"{mode} {name}={raw!r} is not an integer") from exc - if value < 1: - raise ValueError(f"{mode} {name}={value} must be >= 1") - return value - - def _tp_size(self, mode: WorkerMode) -> int: - return self._get_int_flag(mode, "tensor-parallel-size", 1) - - def _pp_size(self, mode: WorkerMode) -> int: - return self._get_int_flag(mode, "pipeline-parallel-size", 1) - - def _gpus_per_dp_rank(self, mode: WorkerMode) -> int: - """GPUs consumed by one DP rank: TP × PP.""" - return self._tp_size(mode) * self._pp_size(mode) - - def _local_dp_size(self, mode: WorkerMode, gpu_count: int) -> int: - """DP ranks owned by one process that sees ``gpu_count`` GPUs. - - vLLM world size is TP × DP × PP, so a DP rank occupies TP × PP GPUs. - Those GPUs must fit on the node: a DP rank that spans nodes needs the - non-DP TP launch path (``--nnodes`` / ``--node-rank``), not DP local ranks. - """ - tp = self._tp_size(mode) - pp = self._pp_size(mode) - gpus_per_rank = tp * pp - if gpu_count % gpus_per_rank != 0: - raise ValueError( - f"{mode} tensor-parallel-size={tp} * pipeline-parallel-size={pp} " - f"= {gpus_per_rank} GPUs per DP rank does not divide the " - f"{gpu_count} GPUs allocated on each node" - ) - return gpu_count // gpus_per_rank - - def _validate_dp_world_size(self, mode: WorkerMode, dp_size: int, total_gpus: int) -> None: - """Require TP × DP × PP to match the GPUs allocated to the endpoint.""" - tp = self._tp_size(mode) - pp = self._pp_size(mode) - world = dp_size * tp * pp - if world != total_gpus: - raise ValueError( - f"{mode} data-parallel-size={dp_size} * tensor-parallel-size={tp} " - f"* pipeline-parallel-size={pp} = {world} does not match " - f"the endpoint's {total_gpus} allocated GPUs" - ) - def _dp_rank_gpu_groups(self, mode: WorkerMode, gpu_indices: frozenset[int]) -> list[frozenset[int]]: """Split a node's GPUs into one group per local DP rank.""" - gpus_per_rank = self._gpus_per_dp_rank(mode) - self._local_dp_size(mode, len(gpu_indices)) + gpus_per_rank = self._get_model_parallel_size(mode) + self._get_local_dp_size(mode, len(gpu_indices)) sorted_gpus = sorted(gpu_indices) return [ frozenset(sorted_gpus[offset : offset + gpus_per_rank]) @@ -985,7 +929,7 @@ def _dp_per_gpu_endpoints_to_processes( sidecar_grpc: bool, every_rank_serves: bool = False, ) -> list[Process]: - """DP+EP mode with one process per DP rank (TP x PP GPUs each). + """DP+EP mode with one process per DP rank (TP x PP x PCP GPUs each). Only rank 0 serves the API unless ``every_rank_serves`` (external load balancing), when every rank gets its own HTTP port. @@ -1015,12 +959,10 @@ def _dp_per_gpu_endpoints_to_processes( ) continue - # One process per DP rank. With TP=1 this is one GPU; with TP>1 the - # process owns the TP x PP GPUs for that rank. + # One process per DP rank, owning its TP x PP x PCP GPUs. dp_rank = 0 dp_rpc_port = allocator.next(DP_RPC_PORTS, endpoint.leader_node) - dp_size = self._get_dp_size(endpoint.mode) or (endpoint.total_gpus // self._gpus_per_dp_rank(endpoint.mode)) - self._validate_dp_world_size(endpoint.mode, dp_size, endpoint.total_gpus) + dp_size, _ = self._validate_endpoint_parallelism(endpoint) rank_gpu_groups = self._dp_rank_gpu_groups(endpoint.mode, endpoint.gpu_indices) # vLLM computes actual_port = base + data_parallel_rank, so every DP # rank of the endpoint shares one reserved block. diff --git a/tests/test_vllm_dp_parallelism.py b/tests/test_vllm_dp_parallelism.py new file mode 100644 index 000000000..e199b2d12 --- /dev/null +++ b/tests/test_vllm_dp_parallelism.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""DP rank placement includes every model-parallel dimension.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from srtctl.backends import VLLMBackend +from srtctl.core.schema import RoleConfig +from srtctl.core.topology import Endpoint + + +@pytest.mark.parametrize("frontend", ["llm-d", "sglang-router", "dynamo"]) +@pytest.mark.parametrize("node_count", [1, 2]) +@pytest.mark.parametrize("tp,pp,pcp", [(1, 1, 2), (2, 1, 2), (1, 2, 2)]) +def test_dp_ranks_own_complete_model_parallel_groups(frontend, node_count, tp, pp, pcp): + replica_size = tp * pp * pcp + backend = VLLMBackend( + dp_launch_mode="per_gpu", + roles={ + "agg": RoleConfig( + args={ + "data-parallel-size": 2 * node_count, + "tensor-parallel-size": tp, + "pipeline-parallel-size": pp, + "prefill-context-parallel-size": pcp, + "data-parallel-external-lb": True, + } + ) + }, + ) + endpoint = Endpoint("agg", 0, tuple(f"node{i}" for i in range(node_count)), frozenset(range(2 * replica_size))) + processes = backend.endpoints_to_processes([endpoint], frontend_type=frontend) + + assert len(processes) == 2 * node_count + assert [p.dp_rank for p in processes] == list(range(2 * node_count)) + for node in endpoint.nodes: + ranks = [p for p in processes if p.node == node] + assert [p.gpu_indices for p in ranks] == [ + frozenset(range(replica_size)), + frozenset(range(replica_size, 2 * replica_size)), + ] + if frontend != "dynamo": + assert all(p.http_port > 0 for p in processes) + assert len({(p.node, p.http_port) for p in processes}) == len(processes) + runtime = MagicMock(model_path=Path("/model"), is_hf_model=False, frontend_port=8000) + with patch("srtctl.core.slurm.get_hostname_ip", return_value="127.0.0.1"): + for process in processes: + command = backend.build_worker_command( + process=process, endpoint_processes=processes, runtime=runtime, frontend_type=frontend + ) + assert command[command.index("--data-parallel-rank") + 1] == str(process.dp_rank) + assert command[command.index("--prefill-context-parallel-size") + 1] == str(pcp) + assert "--headless" not in command + + +@pytest.mark.parametrize( + "dp,pcp,gpu_count,node_count,error", [(2, 4, 2, 1, "require 8 GPUs"), (3, 2, 3, 2, "not divisible")] +) +def test_per_rank_launch_rejects_invalid_pcp_allocation(dp, pcp, gpu_count, node_count, error): + backend = VLLMBackend( + roles={ + "agg": RoleConfig( + args={ + "data-parallel-size": dp, + "prefill-context-parallel-size": pcp, + "data-parallel-external-lb": True, + } + ) + } + ) + nodes = tuple(f"node{i}" for i in range(node_count)) + # The second case has the correct total world size but cannot fit a rank on each node. + endpoint = Endpoint("agg", 0, nodes, frozenset(range(gpu_count))) + with pytest.raises(ValueError, match=error): + backend.endpoints_to_processes([endpoint], frontend_type="llm-d") From c00ec8504dfe5c81191924b7e61aa3510ba8822f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 10:48:39 -0500 Subject: [PATCH 11/25] fix(tests): distinguish subscriber destinations from listener ports --- tests/test_design_rule_port_arithmetic.py | 64 +++++++++++++++++++ tests/test_design_rules.py | 75 ++++++++++++++++++++++- 2 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 tests/test_design_rule_port_arithmetic.py diff --git a/tests/test_design_rule_port_arithmetic.py b/tests/test_design_rule_port_arithmetic.py new file mode 100644 index 000000000..87c53c270 --- /dev/null +++ b/tests/test_design_rule_port_arithmetic.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Outbound subscriber offsets must not hide derived listener ports.""" + +import ast +import textwrap + +import pytest + +from tests.test_design_rules import port_arithmetic + + +def _violations(body: str) -> set[str]: + tree = ast.parse("def configure(frontend, process, runtime, model, rank):\n" + textwrap.indent(body, " ")) + return {site for _, site in port_arithmetic("backends/example.py", tree)} + + +@pytest.mark.parametrize( + "holder", ["", "target = frontend.kv_events_subscriber(process, runtime, model) if rank else None\n"] +) +@pytest.mark.parametrize("operation", ["+", "-"]) +def test_subscriber_destination_offset_is_not_a_listener(holder: str, operation: str) -> None: + source = "target" if holder else "frontend.kv_events_subscriber(process, runtime, model)" + assert not _violations( + holder + + f"remote_host, remote_port, topic = {source}\n" + + f'config = {{"endpoint": f"tcp://{{remote_host}}:{{remote_port {operation} rank}}"}}\n' + ) + + +@pytest.mark.parametrize( + "body", + [ + "socket.bind((remote_host, remote_port - rank))", + "listener_port = remote_port + 1", + 'config = {"endpoint": f"tcp://*:{remote_port - rank}"}', + 'config = {"endpoint": f"tcp://{local_host}:{remote_port - rank}"}', + 'config = {"listen": f"tcp://{remote_host}:{remote_port - rank}"}', + 'remote_host = "0.0.0.0"\nconfig = {"endpoint": f"tcp://{remote_host}:{remote_port - rank}"}', + 'remote_port = 8000\nconfig = {"endpoint": f"tcp://{remote_host}:{remote_port - rank}"}', + ], +) +def test_subscriber_port_does_not_exempt_listener_or_reassigned_address(body: str) -> None: + assert _violations( + "remote_host, remote_port, topic = frontend.kv_events_subscriber(process, runtime, model)\n" + body + ) + + +def test_unknown_endpoint_source_is_still_checked() -> None: + assert _violations( + 'host, port, topic = frontend.worker_address(process)\nconfig = {"endpoint": f"tcp://{host}:{port - rank}"}\n' + ) == {"port - rank"} + + +def test_subscriber_binding_does_not_leak_into_another_function() -> None: + tree = ast.parse( + "def publisher(frontend):\n" + " host, port, topic = frontend.kv_events_subscriber()\n" + ' config = {"endpoint": f"tcp://{host}:{port - rank}"}\n' + "def listener(host, port, rank):\n" + ' config = {"endpoint": f"tcp://{host}:{port - rank}"}\n' + ) + assert list(port_arithmetic("backends/example.py", tree)) == [("backends/example.py", "port - rank")] diff --git a/tests/test_design_rules.py b/tests/test_design_rules.py index e1a6cab6c..6ca88a729 100644 --- a/tests/test_design_rules.py +++ b/tests/test_design_rules.py @@ -88,12 +88,84 @@ def frontend_name_branches(rel: str, tree: ast.Module) -> Iterator[Site]: _PORT_NAME = re.compile(r"(^|_)(port|PORT)(_BASE)?$") +def _subscriber_destination_arithmetic(tree: ast.Module) -> set[ast.BinOp]: + """Recognize outbound KV-event URLs through the Frontend subscriber contract. + + ``kv_events_subscriber`` returns a remote (host, port, topic), not a listener + allocation. A publisher may undo its engine's rank offset in that destination. + Follow single-assignment locals within one function, and exempt only the port + expression in an ``endpoint`` URL using that same remote host. Reusing the + port for a listener, or changing either local, must still trip the rule. + """ + destinations: set[ast.BinOp] = set() + for scope in ast.walk(tree): + if not isinstance(scope, ast.FunctionDef | ast.AsyncFunctionDef): + continue + # Nested scopes have independent bindings and are considered separately. + nodes: list[ast.AST] = [] + pending: list[ast.AST] = list(scope.body) + while pending: + node = pending.pop() + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda): + continue + nodes.append(node) + pending.extend(ast.iter_child_nodes(node)) + stores: dict[str, int] = {} + assignments: dict[str, ast.expr] = {} + for node in nodes: + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store): + stores[node.id] = stores.get(node.id, 0) + 1 + if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + assignments[node.targets[0].id] = node.value + addresses: set[tuple[str, str]] = set() + for node in nodes: + if not (isinstance(node, ast.Assign) and len(node.targets) == 1): + continue + target = node.targets[0] + if not (isinstance(target, ast.Tuple) and len(target.elts) == 3): + continue + host, port, _ = target.elts + if not (isinstance(host, ast.Name) and isinstance(port, ast.Name)): + continue + if stores.get(host.id) != 1 or stores.get(port.id) != 1: + continue + value = node.value + if isinstance(value, ast.Name) and stores.get(value.id) == 1: + value = assignments.get(value.id, value) + if isinstance(value, ast.IfExp) and isinstance(value.orelse, ast.Constant) and value.orelse.value is None: + value = value.body + if ( + isinstance(value, ast.Call) + and isinstance(value.func, ast.Attribute) + and value.func.attr == "kv_events_subscriber" + ): + addresses.add((host.id, port.id)) + for node in nodes: + if not isinstance(node, ast.Dict): + continue + for key, value in zip(node.keys, node.values, strict=True): + if not (isinstance(key, ast.Constant) and key.value == "endpoint" and isinstance(value, ast.JoinedStr)): + continue + match value.values: + case [ + ast.Constant(value="tcp://"), + ast.FormattedValue(value=ast.Name(id=host)), + ast.Constant(value=":"), + ast.FormattedValue(value=ast.BinOp(left=ast.Name(id=port)) as arithmetic), + ] if (host, port) in addresses: + destinations.add(arithmetic) + return destinations + + def port_arithmetic(rel: str, tree: ast.Module) -> Iterator[Site]: if rel in ("ports.py", "core/topology.py"): return + destinations = _subscriber_destination_arithmetic(tree) for node in ast.walk(tree): if not (isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add | ast.Sub)): continue + if node in destinations: + continue for operand in (node.left, node.right): dotted = _dotted(operand) if dotted is not None and _PORT_NAME.search(dotted.rsplit(".", 1)[-1]): @@ -152,9 +224,6 @@ class Rule: frozenset( { ("backends/vllm.py", "grpc_port + 1"), - # Not a listener: a KV-event publisher dials the router's one subscriber port, and vLLM - # adds the publisher's DP rank to the port it is given, so the rank is taken off first. - ("backends/vllm.py", "port - rank"), # KVBM_ZMQ_PORTS allocates a two-port block; the ACK port is the block's second port. ("cli/mixins/worker_stage.py", "leader.kvbm_zmq_port + 1"), } From 313869021f189bcb8e834bc62ec3172c2ac0afce Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 10:54:40 -0500 Subject: [PATCH 12/25] refactor(vllm): share effective KV connector resolution --- docs/llm-d.md | 5 ++- src/srtctl/backends/vllm.py | 61 +++++++++++++++-------------- tests/test_vllm_connectors.py | 72 ++++++++++++++++++++++++++++++++++- 3 files changed, 107 insertions(+), 31 deletions(-) diff --git a/docs/llm-d.md b/docs/llm-d.md index c71728210..cfd952695 100644 --- a/docs/llm-d.md +++ b/docs/llm-d.md @@ -62,8 +62,9 @@ srtctl sets `--port` (the worker's `Process.proxy_port`), `--model-server-port` The allocator hands out the proxy port (`PROXY_PORTS`) for every routable worker of a mode the frontend proxies (`Frontend.proxied_worker_modes`), so two decode workers on one node never collide. The `--kv-connector` protocol comes from the decode workers' vLLM -connector class (`VLLMBackend.kv_connector_classes`, which reads a role's own -`kv-transfer-config` before `connector`): +connector class. Engine commands and sidecar selection use the same payload: +a role's explicit `kv-transfer-config` takes precedence over its `connector`, +which takes precedence over `engine.connector`: | Decode connector | Sidecar protocol | | --- | --- | diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index ea93af832..819c69d4e 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -433,6 +433,19 @@ def connector_for_mode(self, mode: WorkerMode) -> str | None: mode_connector = self.get_config_for_mode(mode).get("connector") return mode_connector if mode_connector is not None else self.connector + def _effective_connector_for_mode(self, mode: WorkerMode, *, use_default: bool = True) -> str | None: + """Resolve explicit KV-transfer JSON before the named connector configuration. + + Direct aggregate workers skip the engine's P/D default, but still honor + an explicit role configuration. Connector aliases retain their service + and discovery metadata independently of raw payload overrides. + """ + config = self.get_config_for_mode(mode) + for key, value in config.items(): + if normalize_vllm_config_key(key) == "kv-transfer-config" and value is not None: + return json.dumps(dict(value)) if isinstance(value, Mapping) else str(value) + return self.connector_for_mode(mode) if use_default else config.get("connector") + def kv_connector_for_mode(self, mode: WorkerMode) -> KVConnector | None: """The connector table row for a mode; None for no connector or a raw JSON ``--kv-transfer-config``.""" return kv_connector_row(self.connector_for_mode(mode)) @@ -445,24 +458,10 @@ def kv_connector_classes(self, mode: WorkerMode) -> tuple[str, ...]: in the role's args wins, as it does on the command line; otherwise the mode's connector (a table row or raw JSON). Empty when the mode runs no connector. """ - explicit = next( - ( - value - for key, value in self.get_config_for_mode(mode).items() - if normalize_vllm_config_key(key) == "kv-transfer-config" - ), - None, - ) - row = self.kv_connector_for_mode(mode) - connector = self.connector_for_mode(mode) - if explicit is not None: - payload = explicit if isinstance(explicit, Mapping) else json.loads(str(explicit)) - elif row is not None: - return (row.kv_connector,) - elif not connector or connector.lower() in ("null", "none"): + connector = self._effective_connector_for_mode(mode) + if not connector or connector.lower() in ("null", "none"): return () - else: - payload = json.loads(connector) + payload = json.loads(_connector_to_kv_transfer_config(connector, mode)) wrapped = (payload.get("kv_connector_extra_config") or {}).get("connectors") or [] return tuple( name for name in (payload.get("kv_connector"), *(inner.get("kv_connector") for inner in wrapped)) if name @@ -478,7 +477,12 @@ def discovers_workers(self) -> bool: return any(row is not None and row.discovery for row in map(self.kv_connector_for_mode, ("prefill", "decode"))) def kv_transfer_config( - self, mode: WorkerMode, process: Process | None = None, runtime: RuntimeContext | None = None + self, + mode: WorkerMode, + process: Process | None = None, + runtime: RuntimeContext | None = None, + *, + use_default: bool = True, ) -> str | None: """``--kv-transfer-config`` JSON for a worker mode, or None when the mode has no connector. @@ -489,7 +493,7 @@ def kv_transfer_config( this worker's HTTP port and routable IP, and the handshake and notify listeners the allocator reserved for it. """ - connector = self.connector_for_mode(mode) + connector = self._effective_connector_for_mode(mode, use_default=use_default) if not connector or connector.lower() in ("null", "none"): return None row = kv_connector_row(connector) @@ -1166,11 +1170,11 @@ def build_worker_command( # A prefill/decode worker gets its KV connector. An aggregate worker gets # only the one its role names (e.g. lmcache-mp offload), not the P/D default. - role_connector = config.pop("connector", None) - if mode in {"prefill", "decode"} or role_connector is not None: - kv_transfer_config = self.kv_transfer_config(mode, process, runtime) - if kv_transfer_config is not None: - config.setdefault("kv-transfer-config", kv_transfer_config) + kv_transfer_config = self.kv_transfer_config(mode, process, runtime, use_default=mode != "agg") + for key in ("connector", "kv-transfer-config", "kv_transfer_config"): + config.pop(key, None) + if kv_transfer_config is not None: + config["kv-transfer-config"] = kv_transfer_config node_rank = endpoint_nodes.index(process.node) # The worker that is itself the public endpoint may run the alternate @@ -1328,7 +1332,8 @@ def build_worker_command( # KV connector → --kv-transfer-config (dynamo 1.0.0+: --connector was removed). # Pop from config so it doesn't get added again by _config_to_cli_args. - config.pop("connector", None) + for key in ("connector", "kv-transfer-config", "kv_transfer_config"): + config.pop(key, None) kv_transfer_cfg = self.kv_transfer_config(mode, process, runtime) if kv_transfer_cfg is not None: cmd.extend(["--kv-transfer-config", kv_transfer_cfg]) @@ -1621,9 +1626,9 @@ def _build_sidecar_command( if hybrid_lb: command.extend(["--data-parallel-start-rank", str(process.node_rank), "--data-parallel-hybrid-lb"]) - config.pop("connector", None) - has_explicit_kv = "kv-transfer-config" in config or "kv_transfer_config" in config - kv_transfer_cfg = None if has_explicit_kv else self.kv_transfer_config(mode) + for key in ("connector", "kv-transfer-config", "kv_transfer_config"): + config.pop(key, None) + kv_transfer_cfg = self.kv_transfer_config(mode) if kv_transfer_cfg is not None: command.extend(["--kv-transfer-config", kv_transfer_cfg]) diff --git a/tests/test_vllm_connectors.py b/tests/test_vllm_connectors.py index 350664bfa..3a195c5a9 100644 --- a/tests/test_vllm_connectors.py +++ b/tests/test_vllm_connectors.py @@ -91,8 +91,8 @@ def test_direct_aggregate_worker_runs_only_its_role_connector(aggregated, expect from pathlib import Path from unittest.mock import MagicMock - from srtctl.core.topology import Process from srtctl.core.schema import RoleConfig + from srtctl.core.topology import Process backend = VLLMBackend(connector="nixl", roles={"agg": RoleConfig(args=aggregated)}) process = Process( @@ -113,3 +113,73 @@ def test_direct_aggregate_worker_runs_only_its_role_connector(aggregated, expect assert "--connector" not in cmd kv_config = json.loads(cmd[cmd.index("--kv-transfer-config") + 1]) if "--kv-transfer-config" in cmd else None assert kv_config == expected + + +@pytest.mark.parametrize("frontend,sidecar", [("llm-d", False), ("dynamo", False), ("dynamo", True)]) +@pytest.mark.parametrize("mode", ["agg", "decode"]) +@pytest.mark.parametrize("key", ["kv-transfer-config", "kv_transfer_config"]) +@pytest.mark.parametrize("as_mapping", [False, True]) +def test_explicit_transfer_config_is_shared_by_commands_and_sidecar(frontend, sidecar, mode, key, as_mapping): + """An explicit MultiConnector beats both aliases and reaches each engine exactly once.""" + import shlex + from pathlib import Path + from unittest.mock import MagicMock, patch + + from srtctl.core.schema import DynamoConfig + from srtctl.core.topology import Endpoint + from srtctl.services.llm_d_sidecar import sidecar_kv_connector + + payload = { + "kv_connector": "MultiConnector", + "kv_role": "kv_both", + "kv_connector_extra_config": { + "connectors": [{"kv_connector": "OffloadingConnector"}, {"kv_connector": "NixlConnector"}] + }, + } + explicit = payload if as_mapping else json.dumps(payload) + backend = VLLMBackend( + connector="lmcache-mp", + roles={mode: RoleConfig(args={"connector": "lmcache", key: explicit})}, + ) + assert json.loads(backend.kv_transfer_config(mode)) == payload + assert backend.kv_connector_classes(mode) == ("MultiConnector", "OffloadingConnector", "NixlConnector") + assert backend.kv_connector_for_mode(mode) is _CONNECTOR_MAP["lmcache"] + if mode == "decode": + assert sidecar_kv_connector(backend) == "nixlv2" + endpoint = Endpoint(mode, 0, ("node0",), frozenset({0})) + processes = backend.endpoints_to_processes([endpoint], frontend_type=frontend, dynamo_sidecar=sidecar) + assert all(p.moriio_handshake_port is None for p in processes) + runtime = MagicMock(model_path=Path("/model"), is_hf_model=False, frontend_port=9000) + runtime.dynamo = DynamoConfig(sidecar=sidecar) + with patch("srtctl.core.slurm.get_hostname_ip", return_value="127.0.0.1"): + command = backend.build_worker_command( + process=processes[0], endpoint_processes=processes, runtime=runtime, frontend_type=frontend + ) + if sidecar: + command = shlex.split( + next(line.removesuffix(" &") for line in command[2].splitlines() if line.startswith("vllm-rs serve ")) + ) + assert command.count("--kv-transfer-config") == 1 + assert json.loads(command[command.index("--kv-transfer-config") + 1]) == payload + assert "--connector" not in command + assert backend.get_config_for_mode(mode)[key] == explicit + + +def test_raw_override_preserves_named_connectors_implied_service(): + """Customizing the LMCache payload does not drop the alias's local server dependency.""" + from pathlib import Path + + import yaml + + from srtctl.core.schema import SrtConfig + from srtctl.services.implicit import connector_services + + payload = _CONNECTOR_MAP["lmcache-mp"].transfer_config("agg") + payload["kv_connector_extra_config"]["custom_option"] = True + recipe = yaml.safe_load(Path("examples/features/lmcache-server.yaml").read_text()) + recipe.pop("services") + recipe["roles"]["agg"]["args"]["kv-transfer-config"] = payload + config = SrtConfig.Schema().load(recipe) + + assert [entry.service.type for entry in connector_services(config)] == ["lmcache-server"] + assert json.loads(config.backend.kv_transfer_config("agg")) == payload From 41616adc00ecce70f77621c68c2e28a690a10494 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 12:06:40 -0500 Subject: [PATCH 13/25] refactor(llm-d): rename the Envoy template to llm_d_envoy.yaml.j2 Signed-off-by: Cam Quilici --- src/srtctl/templates/{llm-d-envoy.yaml.j2 => llm_d_envoy.yaml.j2} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/srtctl/templates/{llm-d-envoy.yaml.j2 => llm_d_envoy.yaml.j2} (100%) diff --git a/src/srtctl/templates/llm-d-envoy.yaml.j2 b/src/srtctl/templates/llm_d_envoy.yaml.j2 similarity index 100% rename from src/srtctl/templates/llm-d-envoy.yaml.j2 rename to src/srtctl/templates/llm_d_envoy.yaml.j2 From 9fe40b6b05ca15b39f20d57ae7c53786adea518a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 12:17:58 -0500 Subject: [PATCH 14/25] fix(llm-d): load the renamed Envoy template --- src/srtctl/frontends/llm_d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py index afea9b3db..9151f7b7a 100644 --- a/src/srtctl/frontends/llm_d.py +++ b/src/srtctl/frontends/llm_d.py @@ -212,7 +212,7 @@ def envoy_config(listen_port: int, access_log: str) -> str: from jinja2 import Environment, FileSystemLoader templates = Environment(loader=FileSystemLoader(str(Path(__file__).parent.parent / "templates"))) - return templates.get_template("llm-d-envoy.yaml.j2").render( + return templates.get_template("llm_d_envoy.yaml.j2").render( admin_port=LLM_D_ENVOY_ADMIN_PORT, listen_port=listen_port, access_log=access_log, From 35637e8a2146c4a91f2655c5ca39061a7f79d613 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 16:25:47 -0500 Subject: [PATCH 15/25] docs(llm-d): tighten implementation comments --- src/srtctl/backends/vllm.py | 23 +++----- src/srtctl/core/topology.py | 9 +-- src/srtctl/core/worker_backends.py | 2 +- src/srtctl/frontends/base.py | 13 ++--- src/srtctl/frontends/llm_d.py | 73 ++++++++++-------------- src/srtctl/frontends/static_router.py | 6 +- src/srtctl/ports.py | 13 ++--- src/srtctl/services/llm_d_sidecar.py | 24 ++++---- src/srtctl/services/registry.py | 9 +-- src/srtctl/templates/llm_d_envoy.yaml.j2 | 13 ++--- 10 files changed, 67 insertions(+), 118 deletions(-) diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index 819c69d4e..cb60a0c5c 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -451,12 +451,10 @@ def kv_connector_for_mode(self, mode: WorkerMode) -> KVConnector | None: return kv_connector_row(self.connector_for_mode(mode)) def kv_connector_classes(self, mode: WorkerMode) -> tuple[str, ...]: - """The vLLM connector classes a mode's workers run: ``kv_connector`` of their ``--kv-transfer-config``. + """Connector classes from the effective transfer config; empty without a connector. - A ``MultiConnector`` is followed by the classes of the connectors it wraps - (``kv_connector_extra_config.connectors``). A ``kv-transfer-config`` written - in the role's args wins, as it does on the command line; otherwise the mode's - connector (a table row or raw JSON). Empty when the mode runs no connector. + Include wrapped classes after ``MultiConnector``. Resolution follows + ``_effective_connector_for_mode``, matching worker command generation. """ connector = self._effective_connector_for_mode(mode) if not connector or connector.lower() in ("null", "none"): @@ -700,12 +698,7 @@ def _is_dp_mode(self, mode: WorkerMode) -> bool: return dp_size is not None and int(dp_size) > 1 def is_external_lb(self, mode: WorkerMode) -> bool: - """Whether a DP mode's ranks are each an API server of their own (``data-parallel-external-lb``). - - A router then addresses every rank: each is its own ``vllm serve - --data-parallel-rank`` process on its own HTTP port (vLLM's external load - balancing) instead of one server that balances its ranks itself. - """ + """Whether ``data-parallel-external-lb`` exposes each DP rank as a separate HTTP server.""" return self._is_dp_mode(mode) and any( normalize_vllm_config_key(key) == "data-parallel-external-lb" and value for key, value in self.get_config_for_mode(mode).items() @@ -872,9 +865,8 @@ def endpoints_to_processes( allocator = port_allocator_for(port_allocator, base_sys_port) frontend = get_frontend(frontend_type) if frontend.worker_launch == "direct" and not frontend.expands_node_local_dp: - # One `vllm serve` owns every DP rank of the endpoint, and only the - # leader serves the API, so the standard topology applies; behind a - # router, an external-LB endpoint is one `vllm serve` per rank instead. + # Direct serving normally exposes one API per endpoint. External load + # balancing behind a router requires a separate server per DP rank. processes = [] for endpoint in endpoints: if self.is_external_lb(endpoint.mode) and frontend.worker_api_port(endpoint.mode) == "allocated": @@ -1305,8 +1297,7 @@ def build_worker_command( subscriber = frontend.kv_events_subscriber(process, runtime, served_model_name) if kv_events else None if kv_events and subscriber is not None: host, port, topic = subscriber - # vLLM adds the publisher's DP rank to the port it is given; start that far - # below so every rank connects to the one subscriber. + # Offset vLLM's rank addition so every publisher reaches the same subscriber port. rank = process.dp_rank or 0 kv_events = {**kv_events, "endpoint": f"tcp://{host}:{port - rank}", "topic": topic} cmd.extend(["--kv-events-config", json.dumps(kv_events)]) diff --git a/src/srtctl/core/topology.py b/src/srtctl/core/topology.py index addbfd16e..16b828ff6 100644 --- a/src/srtctl/core/topology.py +++ b/src/srtctl/core/topology.py @@ -164,8 +164,7 @@ class Process: their own ports and their own srun step. kvbm_zmq_port: KVBM leader ZMQ pub port (ack is the next port); the leader's is used sidecar_grpc_port: Dynamo sidecar gRPC listener, allocated when the job runs sidecars - proxy_port: listener of the proxy the router sends this routable worker's traffic - through (llm-d's P/D sidecar), allocated when the frontend proxies its mode + proxy_port: Per-worker proxy listener (e.g. llm-d's decode sidecar), when required by the frontend nccl_port: SGLang local TP rendezvous port, one per server process dist_init_port: SGLang multi-node dist-init port; the same value on every process of an endpoint grpc_http_port: HTTP sidecar (/metrics, profiler routes) of an SGLang gRPC-mode leader @@ -213,11 +212,7 @@ def is_leader(self) -> bool: @property def engine_suffix(self) -> str: - """Step-name and log-name suffix that tells a worker's processes on one node apart. - - ``_dp`` for one DP rank of a per-rank endpoint, ``_e`` for a shadow - engine; ``""`` for the only process. - """ + """Disambiguate colocated processes with optional ``_dp`` and ``_e`` suffixes.""" rank = f"_dp{self.dp_rank}" if self.dp_rank is not None else "" return f"{rank}_e{self.engine_id}" if self.engine_id else rank diff --git a/src/srtctl/core/worker_backends.py b/src/srtctl/core/worker_backends.py index 7e6db0683..b8c94f91b 100644 --- a/src/srtctl/core/worker_backends.py +++ b/src/srtctl/core/worker_backends.py @@ -77,7 +77,7 @@ def expand(backend: BackendConfig, selected: list[Endpoint]) -> list[Process]: def with_proxy_ports(config: SrtConfig, processes: list[Process], allocator: NodePortAllocator) -> list[Process]: - """Give every routable worker of a mode the frontend proxies the listener its proxy binds.""" + """Allocate proxy ports for routable workers in the frontend's proxied modes.""" from srtctl.frontends import FRONTEND_NONE, get_frontend if config.frontend.type == FRONTEND_NONE: diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py index a0805ca5a..7a0191ce5 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py @@ -181,22 +181,19 @@ def implied_services(self, config: Any) -> list["EffectiveService"]: return [] def proxied_worker_modes(self, config: Any) -> frozenset[str]: - """Worker modes the router reaches through a per-worker proxy instead of the worker's own port. + """Worker modes reached through a per-worker proxy. - Every routable worker of these modes gets ``Process.proxy_port`` from the - allocator (``core.worker_backends.worker_processes``); the frontend implies - the service that binds it (llm-d: its P/D sidecar on decode workers). + ``worker_processes`` allocates ``Process.proxy_port`` for each routable + worker of these modes; the frontend must imply the service that binds it. """ return frozenset() def kv_events_subscriber( self, process: "Process", runtime: "RuntimeContext", model_name: str ) -> tuple[str, int, str] | None: - """Where a worker publishes its KV-cache events for this router: ``(host, port, topic)``. + """KV-event destination ``(host, port, topic)``, or ``None`` to leave direct workers unconfigured. - ``None`` when the router subscribes to none; a role's ``kv_events`` then - configures nothing on a direct worker. The engine connects its publisher - to ``host:port`` (adjusting for any per-rank port offset of its own). + Backends must compensate for any engine-added rank offset to reach this port. """ return None diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py index 9151f7b7a..49de21c01 100644 --- a/src/srtctl/frontends/llm_d.py +++ b/src/srtctl/frontends/llm_d.py @@ -4,26 +4,16 @@ """llm-d router frontend (`frontend.type: llm-d`): the Endpoint Picker behind Envoy. -llm-d routes with two processes on the router node. Envoy owns the public port -and asks the Endpoint Picker (EPP) over ext_proc which worker each request goes -to; the EPP answers with the ``x-gateway-destination-endpoint`` header and -Envoy's ORIGINAL_DST cluster forwards there. For prefill/decode the EPP picks a -decode endpoint and names the prefill worker in ``x-prefiller-host-port``; that -decode endpoint is llm-d's P/D sidecar, a service this frontend implies on every -routable decode worker (``services/llm_d_sidecar.py``). - -Without Kubernetes the EPP reads its workers from a file (the ``file-discovery`` -plugin), so srtctl is the registrar: once every worker answers ``/health`` it -writes the endpoints file, the EPP configuration (the recipe's -``frontend.epp_config`` plus the discovery plugin), and the Envoy configuration -into the log directory, then starts the EPP and Envoy. The job is ready when -Envoy's admin ``/ready`` answers and the EPP reports every endpoint ready -(``llm_d_epp_ready_endpoints``: endpoints whose metrics it scrapes). - -An external-LB DP rank (``data-parallel-external-lb``) is an endpoint of its own. -With ``roles..kv_events`` the workers publish their KV-cache events to the -EPP's precise prefix-cache producer on one socket srtctl owns -(``kv_events_subscriber``, ``with_kv_events``). +Envoy accepts HTTP requests and asks the Endpoint Picker (EPP) to select workers +via gRPC ext_proc. It forwards to ``x-gateway-destination-endpoint`` using +ORIGINAL_DST. For P/D, EPP selects a decode sidecar and passes the prefill worker +in ``x-prefiller-host-port``; see ``services/llm_d_sidecar.py``. + +srtctl supplies a fixed endpoint list through EPP's file-discovery plugin, +including one endpoint per external-LB DP rank. Startup waits for worker health +when enabled, writes the configurations, then launches EPP and Envoy. Readiness +requires Envoy's ``/ready`` and EPP's ready-endpoint count, based on worker metrics. +Optional ``roles..kv_events`` feed EPP's precise prefix-cache index. Upstream, pinned: llm-d-router v0.11.0 (https://github.com/llm-d/llm-d-router/tree/a5cbe600ebade00cf3e9885beaf2bfacddeabce1): @@ -80,9 +70,8 @@ # Gauge of endpoints whose metrics the EPP scraped within its staleness window # (pkg/epp/metrics/llm_d_router_metrics.go). READY_ENDPOINTS_METRIC = "llm_d_epp_ready_endpoints" -# Precise prefix-cache routing: the producer indexes the workers' KV-cache events, which -# it receives on one ZMQ socket srtctl owns; the token producer tokenizes each prompt the -# way the engine does, through a worker's /v1/*/render endpoints +# Precise routing indexes KV-cache events and uses a worker's /v1/*/render endpoints +# to obtain matching token IDs # (pkg/epp/framework/plugins/requestcontrol/dataproducer/{preciseprefixcache,tokenizer}). PRECISE_PREFIX_PRODUCER = "precise-prefix-cache-producer" TOKEN_PRODUCER = "token-producer" @@ -101,19 +90,17 @@ def _is_pd(config: Any) -> bool: def _routed_port(process: Process) -> int: - """The port the EPP routes a worker's traffic to: its proxy's when it has one, else its own.""" + """Route through the worker's proxy when present, otherwise its HTTP API.""" return process.proxy_port if process.proxy_port is not None else process.http_port def _container_path(name: str) -> str: - """``log_dir/`` as every container sees it (``log_dir`` is mounted at ``/logs``).""" + """Container path for a file in ``log_dir``, mounted at ``/logs``.""" return str(Path("/logs") / name) -# Scorers and weights of llm-d's no-Kubernetes guide -# (guides/no-kubernetes-deployment/router/epp/config.yaml), the scheduler srtctl runs when a -# recipe gives no frontend.epp_config. A prefill/decode job scores prefill workers with all -# of them and decode workers by load alone. +# Defaults from guides/no-kubernetes-deployment/router/epp/config.yaml. +# In P/D mode, prefill uses all scorers; decode uses only load scorers. _DEFAULT_SCORERS = { "queue-scorer": 2, "kv-cache-utilization-scorer": 2, @@ -124,7 +111,7 @@ def _container_path(name: str) -> str: def default_epp_config(pd: bool) -> dict[str, Any]: - """The scheduler for a recipe without ``frontend.epp_config``: the guide's scorers, split into P/D profiles for ``pd``.""" + """Default scheduler when ``frontend.epp_config`` is unset, with separate profiles for P/D.""" def profile(name: str, scorers: Any, role_filter: str | None = None) -> dict[str, Any]: refs = [{"pluginRef": scorer, "weight": _DEFAULT_SCORERS[scorer]} for scorer in scorers] @@ -182,12 +169,11 @@ def _vllm_token_producer(plugin: dict[str, Any]) -> bool: def with_kv_events(document: dict[str, Any], render_url: str, model_name: str) -> dict[str, Any]: - """Point the precise prefix-cache producer at srtctl's KV-event socket and the token producer at a worker. + """Configure the KV-event subscriber and the worker URL used for tokenization. - The EPP binds the socket and every publishing worker connects to it (the - producer's global-socket mode, ``kvEventsConfig.zmqEndpoint``): its per-pod - mode dials one fixed port per endpoint address, which workers sharing a node - cannot all bind. vLLM token producers render through ``render_url``. + Workers connect to one EPP socket (``kvEventsConfig.zmqEndpoint``). The + alternative per-pod mode requires a fixed worker port, which collides when + workers share a node. """ for plugin in document.get("plugins") or []: parameters = _parameters(plugin) @@ -239,7 +225,7 @@ class LLMDFrontend(StaticRouterFrontend): # The EPP takes prefill/decode from its scheduler configuration, not a flag. pd_flag: ClassVar[str] = "" process_name: ClassVar[str] = "llm-d" - # The endpoints file is written once, from workers that already answer /health. + # Gate startup on worker health unless health checks are disabled. wait_for_workers_before_start: ClassVar[bool] = True def validate(self, config: Any) -> None: @@ -335,17 +321,16 @@ def _validate_kv_events(self, config: Any) -> None: ) def kv_events_subscriber(self, process: Process, runtime: RuntimeContext, model_name: str) -> tuple[str, int, str]: - """The EPP's KV-event socket on the head node; the topic names the endpoint the events belong to. + """Return the EPP socket and ``kv@@`` topic. - vLLM's topic is ``kv@@``: the producer files the events under - that endpoint's ``address:port`` (the sidecar's port for a proxied decode - worker) and the model the requests name (pkg/kvevents/engineadapter/vllm_adapter.go). + Use the routed endpoint, including the sidecar port, to match EPP's cache + index keys (pkg/kvevents/engineadapter/vllm_adapter.go). """ address = self.resolve_worker_host(process.node, runtime.network_interface) return runtime.head_node_ip, LLM_D_EPP_KV_EVENTS_PORT, f"kv@{address}:{_routed_port(process)}@{model_name}" def worker_metrics_port(self, process: Process, runtime: RuntimeContext) -> int | None: - """Every routable worker process serves its own API and metrics, an external-LB DP rank included.""" + """Scrape each routable worker directly, including external-LB DP ranks.""" return process.http_port if process.http_port > 0 else None def worker_endpoint_port(self, process: Process, config: Any, runtime: RuntimeContext) -> int | None: @@ -377,7 +362,7 @@ def frontend_metrics_port(self, frontend_args: dict[str, Any] | None) -> int | N return LLM_D_EPP_METRICS_PORT def health_expectations(self, config: Any, processes: list[Process] | None) -> tuple[int, int, str]: - """One endpoint per routable worker process: its prefill vLLM, its decode sidecar, or its aggregate vLLM.""" + """Count routable processes, including individual external-LB DP ranks.""" if processes is None: return super().health_expectations(config, processes) routable = [process for process in processes if process.http_port > 0] @@ -424,7 +409,7 @@ def endpoints_document( return {"endpoints": endpoints} def render_url(self, backend_processes: list[Process], network_interface: str | None) -> str: - """A worker's own API (not its proxy), for the token producer: the first prefill worker, else the first.""" + """Direct worker API for tokenization; prefer the first prefill worker.""" routable = [process for process in backend_processes if process.http_port > 0] process = next((p for p in routable if p.endpoint_mode == "prefill"), routable[0]) return f"http://{self.resolve_worker_host(process.node, network_interface)}:{process.http_port}" @@ -482,7 +467,7 @@ def start_frontends( commands = { "epp": self.epp_command(config, _container_path(EPP_CONFIG_FILE)), - # One Envoy per step: no hot-restart sockets shared with another Envoy on the host network. + # Avoid hot-restart socket collisions between Envoy instances on the host. "envoy": ["envoy", "-c", _container_path(ENVOY_CONFIG_FILE), "--disable-hot-restart"], } container_image = config.frontend.container_image or str(runtime.container_image) diff --git a/src/srtctl/frontends/static_router.py b/src/srtctl/frontends/static_router.py index 34ff03ddc..bb1f0ef7e 100644 --- a/src/srtctl/frontends/static_router.py +++ b/src/srtctl/frontends/static_router.py @@ -227,11 +227,9 @@ def get_backend_health_urls( def wait_for_workers( self, workers: list[RouterWorker], config: Any, stop_event: threading.Event | None = None ) -> None: - """Block until every advertised worker is healthy, within the recipe's health-check budget. + """Wait for HTTP ``/health`` or gRPC TCP connections; skip when health checks are disabled. - HTTP workers must answer ``/health`` with 200; a gRPC worker serves no HTTP - ``/health``, so its port accepting connections is the signal. Raises when - the budget runs out or ``stop_event`` is set. + Raise on cancellation or exhaustion of the recipe's health-check budget. """ if config.health_check is None: return diff --git a/src/srtctl/ports.py b/src/srtctl/ports.py index 8790b23f9..acf1ac57d 100644 --- a/src/srtctl/ports.py +++ b/src/srtctl/ports.py @@ -42,18 +42,14 @@ # passed as --prometheus-port so srtctl, not upstream's default, owns it. SMG_METRICS_PORT = 29000 -# llm-d router (frontend.type: llm-d) on each router node: the Endpoint Picker's ext_proc gRPC, -# gRPC health, and Prometheus listeners (upstream's defaults, passed explicitly so srtctl owns -# them) and Envoy's admin listener, which serves /ready. +# llm-d router-node listeners, passed explicitly; Envoy admin serves /ready. LLM_D_EPP_GRPC_PORT = 9002 LLM_D_EPP_HEALTH_PORT = 9003 LLM_D_EPP_METRICS_PORT = 9090 LLM_D_ENVOY_ADMIN_PORT = 9901 -# The EPP's ZMQ subscriber for vLLM KV-cache events (precise prefix-cache routing): it binds -# this port on the router node and every publishing worker connects to it (upstream's 5557). +# EPP binds this ZMQ subscriber; all KV-event publishers connect to it. LLM_D_EPP_KV_EVENTS_PORT = 5557 -# First listener of a per-worker proxy a router sends a worker's traffic through -# (llm-d's P/D sidecar in front of a decode worker); see PROXY_PORTS. +# Per-worker proxy listeners, including llm-d decode sidecars; see PROXY_PORTS. WORKER_PROXY_PORT_BASE = 9600 # TRT-LLM torch.distributed bootstrap, one port per MPI endpoint. @@ -139,8 +135,7 @@ class PortKind: # KVBM leader ZMQ pair: pub at the port, ack at the port + 1. KVBM_ZMQ_PORTS = PortKind("kvbm_zmq", KVBM_ZMQ_PORT_BASE, 2) SIDECAR_GRPC_PORTS = PortKind("sidecar_grpc", DYNAMO_SIDECAR_GRPC_PORT) -# Bound by the proxy in front of a routable worker whose mode the frontend proxies -# (``Frontend.proxied_worker_modes``), on the worker's node. +# Allocated on each worker's node for Frontend.proxied_worker_modes. PROXY_PORTS = PortKind("proxy", WORKER_PROXY_PORT_BASE, per_node=True) # Engine-specific: the backend allocates these for its own processes. NCCL_PORTS = PortKind("nccl", SGLANG_NCCL_PORT_BASE) diff --git a/src/srtctl/services/llm_d_sidecar.py b/src/srtctl/services/llm_d_sidecar.py index e6edff139..ce75378fa 100644 --- a/src/srtctl/services/llm_d_sidecar.py +++ b/src/srtctl/services/llm_d_sidecar.py @@ -4,18 +4,15 @@ """``type: llm-d-sidecar``: llm-d's P/D sidecar in front of every routable decode worker. -Implied by ``frontend.type: llm-d`` when the job has prefill and decode workers. -The Endpoint Picker routes a request to a decode endpoint and names the prefill -worker in the ``x-prefiller-host-port`` header; the sidecar on that decode -endpoint sends the prefill request first, then hands the returned -``kv_transfer_params`` to its own vLLM, whose KV connector pulls the cache from -the prefill worker. The sidecar is what the router addresses for a decode -worker, so it binds the worker's ``Process.proxy_port`` and proxies everything -else (``/metrics`` included) to the worker's HTTP port. +The llm-d frontend implies this service for P/D jobs. The sidecar calls the +prefill worker named in ``x-prefiller-host-port``, then passes the returned +``kv_transfer_params`` to its local vLLM, which pulls the prefill KV cache. +It listens on ``Process.proxy_port`` and forwards other routes, including +``/metrics``, to the worker's HTTP port. Upstream: llm-d-router ``cmd/pd-sidecar`` and ``pkg/sidecar/proxy`` at v0.11.0 (https://github.com/llm-d/llm-d-router/tree/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/sidecar/proxy). -Its ``GET /health`` answers 200 on its own, so it starts before the workers. +Its ``GET /health`` is independent of vLLM, allowing it to start before workers. """ from __future__ import annotations @@ -45,14 +42,13 @@ def sidecar_kv_connector(backend: Any) -> str: - """The ``--kv-connector`` protocol for the decode workers' KV connector; raises ``ValueError`` when unmapped. + """Select the decode connector's sidecar protocol, or raise ``ValueError``. - A ``MultiConnector`` (P/D transfer next to KV offloading) maps through the first - connector it wraps that has a protocol. + For ``MultiConnector``, use the first supported wrapped connector. """ from srtctl.backends.vllm import VLLMBackend - # The sidecar drives vLLM's KV transfer protocol; another engine's decode has no row. + # Only vLLM's KV-transfer protocol is supported. connectors = backend.kv_connector_classes("decode") if isinstance(backend, VLLMBackend) else () protocol = next((SIDECAR_KV_CONNECTORS[name] for name in connectors if name in SIDECAR_KV_CONNECTORS), None) if protocol is None: @@ -91,7 +87,7 @@ def validate(self, service: ServiceConfig, config: SrtConfig) -> None: ) def container_fallback(self, config: SrtConfig) -> str | None: - """The router image (``frontend.container_image``), which ships the sidecar next to the EPP.""" + """Use the frontend image, which also contains ``pd-sidecar``.""" return config.frontend.container_image def attaches_to(self, process: Process) -> bool: diff --git a/src/srtctl/services/registry.py b/src/srtctl/services/registry.py index 68b56e284..8332eed3f 100644 --- a/src/srtctl/services/registry.py +++ b/src/srtctl/services/registry.py @@ -105,8 +105,6 @@ def template_vars(self) -> dict[str, str]: "worker_gpu_count": str(len(self.process.gpu_indices)), } ) - # The worker's own HTTP API, and the listener of the proxy in front of it - # when the frontend routes through one (Process.proxy_port). if self.process.http_port > 0: values["worker_http_port"] = str(self.process.http_port) if self.process.proxy_port is not None: @@ -203,11 +201,10 @@ def wait_fleet_ready(self, service: ServiceConfig, runtime: RuntimeContext, proc """ def attaches_to(self, process: Process) -> bool: - """Whether a ``placement.per: worker`` instance of this kind attaches to ``process``. + """Filter eligible processes for ``placement.per: worker`` services. - Every engine-0 process of the placed roles by default, followers of a - multi-node worker included; a proxy that fronts a worker's API attaches - only where that API is (``Process.proxy_port``). + By default, attach to all engine-0 processes, including followers. + API proxies override this to exclude processes without a listener. """ return True diff --git a/src/srtctl/templates/llm_d_envoy.yaml.j2 b/src/srtctl/templates/llm_d_envoy.yaml.j2 index 98b257ed5..98c647870 100644 --- a/src/srtctl/templates/llm_d_envoy.yaml.j2 +++ b/src/srtctl/templates/llm_d_envoy.yaml.j2 @@ -1,12 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Envoy in front of the llm-d Endpoint Picker (frontend.type: llm-d), rendered by srtctl. -# Follows llm-d's no-Kubernetes guide: +# Envoy configuration based on llm-d's no-Kubernetes guide: # https://github.com/llm-d/llm-d/blob/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment/router/envoy/envoy.yaml -# Differences: the admin listener binds every interface so srtctl can probe /ready from the -# orchestrator, requests carry no timeout (agentic requests stream for minutes), /metrics is not -# routed, and an access log records the endpoint the EPP picked for every request. +# Allow remote /ready probes and long-running streams; log selected endpoints. admin: address: @@ -39,8 +36,7 @@ static_resources: - name: llm-d domains: ["*"] routes: - # A load-balanced /metrics would be a different worker's counters on every scrape; - # the workers and the EPP serve their own. Answered here, without asking the EPP. + # Scrape workers and EPP directly; load balancing would mix workers' counters. - match: {path: "/metrics"} direct_response: {status: 404} typed_per_filter_config: @@ -75,8 +71,7 @@ static_resources: type: ORIGINAL_DST connect_timeout: 1000s lb_policy: CLUSTER_PROVIDED - # Every request holds a connection here and an ext_proc stream to the EPP for its whole - # lifetime; Envoy's default of 1024 would cap the client's concurrency. + # Each request holds a worker connection and EPP stream; raise the default 1024 concurrency cap. circuit_breakers: thresholds: - {max_connections: 40000, max_pending_requests: 40000, max_requests: 40000} From 2c093c07106af1f128622545f9b248b4fd691fbb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 16:51:44 -0500 Subject: [PATCH 16/25] fix(llm-d): reject frontend.epp_config on other frontends Signed-off-by: Cam Quilici --- src/srtctl/core/schema.py | 2 ++ src/srtctl/frontends/base.py | 3 +++ src/srtctl/frontends/llm_d.py | 1 + tests/test_llm_d_frontend.py | 7 +++++++ 4 files changed, 13 insertions(+) diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 236c48a4c..881ecf0f4 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -3213,6 +3213,8 @@ def _validate_frontend(self) -> None: raise ValidationError( f"frontend.type: {self.frontend.type} requires backend.type: {required}; got {', '.join(incompatible)}" ) + if self.frontend.epp_config is not None and not frontend.accepts_epp_config: + raise ValidationError(f"frontend.epp_config is not supported with frontend.type: {self.frontend.type}") try: frontend.validate(self) except ValueError as exc: diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py index 7a0191ce5..c552fbe25 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py @@ -58,6 +58,9 @@ class Frontend(ABC): #: ``SrtConfig._validate_frontend`` enforces it at config load. required_backend: ClassVar[str | None] = None + #: Whether ``frontend.epp_config`` configures this frontend; ``_validate_frontend`` rejects it otherwise. + accepts_epp_config: ClassVar[bool] = False + #: Role that accepts the public request's model name; None uses decode/agg/prefill order. model_name_role: ClassVar[str | None] = None diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py index 49de21c01..f0fc78719 100644 --- a/src/srtctl/frontends/llm_d.py +++ b/src/srtctl/frontends/llm_d.py @@ -221,6 +221,7 @@ class LLMDFrontend(StaticRouterFrontend): type: ClassVar[str] = "llm-d" required_backend: ClassVar[str | None] = "vllm" + accepts_epp_config: ClassVar[bool] = True executable: ClassVar[tuple[str, ...]] = ("epp",) # The EPP takes prefill/decode from its scheduler configuration, not a flag. pd_flag: ClassVar[str] = "" diff --git a/tests/test_llm_d_frontend.py b/tests/test_llm_d_frontend.py index 6d76a23c8..24115737e 100644 --- a/tests/test_llm_d_frontend.py +++ b/tests/test_llm_d_frontend.py @@ -157,6 +157,13 @@ def test_pd_epp_config_needs_scheduling_profiles() -> None: _load(recipe) +def test_epp_config_needs_the_llm_d_frontend() -> None: + recipe = _recipe() + recipe["frontend"]["type"] = "vllm-router" + with pytest.raises(ValidationError, match="frontend.epp_config is not supported with frontend.type: vllm-router"): + _load(recipe) + + @pytest.mark.parametrize("path", [AGG, DISAGG]) def test_without_epp_config_srtctl_runs_the_guide_scorers(path: Path, tmp_path: Path) -> None: """The EPP's own default only applies to a plugin-less file; srtctl's always carries discovery.""" From 01947da6c32b567bf17ba2929e254db809a24872 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 16:56:42 -0500 Subject: [PATCH 17/25] fix(llm-d): reject unsupported and conflicting configuration --- docs/llm-d.md | 17 +++-- src/srtctl/core/schema.py | 10 +-- src/srtctl/frontends/llm_d.py | 44 +++++++++--- src/srtctl/services/llm_d_sidecar.py | 11 +++ tests/test_llm_d_frontend.py | 101 +++++++++++++++++++++++++-- 5 files changed, 158 insertions(+), 25 deletions(-) diff --git a/docs/llm-d.md b/docs/llm-d.md index cfd952695..9bf1ae223 100644 --- a/docs/llm-d.md +++ b/docs/llm-d.md @@ -58,7 +58,9 @@ services: ``` srtctl sets `--port` (the worker's `Process.proxy_port`), `--model-server-port` (its -`Process.http_port`), `--kv-connector`, and `--secure-proxy=false`; `args` come last. +`Process.http_port`), `--kv-connector`, and `--secure-proxy=false`; overriding these +flags in `command` or `args` is rejected at config load. P/D jobs require exactly one +enabled sidecar service definition; customize the implied entry by name. The allocator hands out the proxy port (`PROXY_PORTS`) for every routable worker of a mode the frontend proxies (`Frontend.proxied_worker_modes`), so two decode workers on one node never collide. The `--kv-connector` protocol comes from the decode workers' vLLM @@ -142,9 +144,11 @@ engine: connector: nixl ``` -`frontend.args` are passed to the EPP; the pool, configuration, and port flags are -srtctl's. A prefill/decode `epp_config` needs `prefill` and `decode` scheduling profiles -and the `disagg-profile-handler`. Without `epp_config`, srtctl runs the scorers of llm-d's +`frontend.epp_config` is accepted only with `frontend.type: llm-d`. +`frontend.args` are passed to the EPP; overriding the pool, configuration, port, or +`secure-serving` flags is rejected at config load. A prefill/decode `epp_config` needs +`prefill` and `decode` scheduling profiles and the `disagg-profile-handler`. +Without `epp_config`, srtctl runs the scorers of llm-d's no-Kubernetes guide ([`config.yaml`](https://github.com/llm-d/llm-d/blob/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment/router/epp/config.yaml)): queue depth (2), KV-cache utilization (2), approximate prefix cache (3), and @@ -234,8 +238,9 @@ default). The roles that publish and the producer go together: either without the other is rejected, as are a producer `kvEventsConfig` that sets the socket or discovery, a -`token-producer` `vllm.url`, and a router not on the head node. A role with -`data-parallel-size` publishes only with `data-parallel-external-lb: true`: each DP rank +`token-producer` `vllm.url`, worker `kv_events.endpoint` or `kv_events.topic`, and a +router not on the head node. The tokenizer URL is managed even without precise routing. +A role with `data-parallel-size` publishes only with `data-parallel-external-lb: true`: each DP rank publishes its own events, and the EPP can place a prefix only on an endpoint that is that rank. diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 881ecf0f4..352755f2f 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -3195,16 +3195,18 @@ def _validate_frontend(self) -> None: schema does not know individual frontends. ``none`` is the services-only job and is covered by ``_validate_services_only``. """ - if self.frontend.type == "none": - return from srtctl.frontends import get_frontend, list_frontend_types try: - frontend = get_frontend(self.frontend.type) + frontend = None if self.frontend.type == "none" else get_frontend(self.frontend.type) except ValueError: raise ValidationError( f"Unknown frontend.type {self.frontend.type!r}. Available: {', '.join(list_frontend_types())}" ) from None + if self.frontend.epp_config is not None and (frontend is None or not frontend.accepts_epp_config): + raise ValidationError(f"frontend.epp_config is not supported with frontend.type: {self.frontend.type}") + if frontend is None: + return required = frontend.required_backend incompatible = [ f"{role}={backend.type}" for role, backend in self.active_role_backends() if backend.type != required @@ -3213,8 +3215,6 @@ def _validate_frontend(self) -> None: raise ValidationError( f"frontend.type: {self.frontend.type} requires backend.type: {required}; got {', '.join(incompatible)}" ) - if self.frontend.epp_config is not None and not frontend.accepts_epp_config: - raise ValidationError(f"frontend.epp_config is not supported with frontend.type: {self.frontend.type}") try: frontend.validate(self) except ValueError as exc: diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py index f0fc78719..577214e08 100644 --- a/src/srtctl/frontends/llm_d.py +++ b/src/srtctl/frontends/llm_d.py @@ -46,7 +46,7 @@ LLM_D_EPP_METRICS_PORT, ) from srtctl.services.config import ServiceConfig, ServicePlacementConfig -from srtctl.services.implicit import EffectiveService +from srtctl.services.implicit import EffectiveService, effective_services from srtctl.services.llm_d_sidecar import LLM_D_SIDECAR_TYPE, sidecar_kv_connector if TYPE_CHECKING: @@ -80,7 +80,16 @@ _OTHER_TOKENIZERS = frozenset({"estimate", "udsTokenizerConfig"}) # EPP flags srtctl sets; frontend.args may not repeat them. _MANAGED_EPP_FLAGS = frozenset( - {"pool-name", "pool-namespace", "config-file", "config-text", "grpc-port", "grpc-health-port", "metrics-port"} + { + "pool-name", + "pool-namespace", + "config-file", + "config-text", + "grpc-port", + "grpc-health-port", + "metrics-port", + "secure-serving", + } ) @@ -231,6 +240,7 @@ class LLMDFrontend(StaticRouterFrontend): def validate(self, config: Any) -> None: frontend = config.frontend + self._validate_epp_args(frontend.args) if frontend.enable_multiple_frontends and config.engine_node_count > 1: raise ValueError( "frontend.type: llm-d requires Envoy and the Endpoint Picker on the public endpoint's node " @@ -248,7 +258,8 @@ def validate(self, config: Any) -> None: epp_config = frontend.epp_config or {} data_layer = epp_config.get("dataLayer") or {} if "discovery" in data_layer or any( - plugin.get("type") == "file-discovery" for plugin in epp_config.get("plugins") or [] + plugin.get("type") == "file-discovery" or plugin.get("name") == DISCOVERY_PLUGIN + for plugin in epp_config.get("plugins") or [] ): raise ValueError( "frontend.epp_config must not configure endpoint discovery: srtctl adds the file-discovery " @@ -262,10 +273,19 @@ def validate(self, config: Any) -> None: "prefill and decode schedulingProfiles and a disaggregation profile handler" ) sidecar_kv_connector(config.backend_for_role("decode")) + sidecars = [entry for entry in effective_services(config) if entry.service.type == LLM_D_SIDECAR_TYPE] + if len(sidecars) != 1: + raise ValueError( + "frontend.type: llm-d with prefill and decode workers requires exactly one enabled " + f"{LLM_D_SIDECAR_TYPE} service definition; override the implied service by name" + ) def _validate_kv_events(self, config: Any) -> None: """Workers publish KV-cache events exactly when the EPP has a precise prefix-cache producer to index them.""" epp_config = config.frontend.epp_config + token_producers = [plugin for plugin in _plugins(epp_config, TOKEN_PRODUCER) if _vllm_token_producer(plugin)] + if any("url" in (_parameters(plugin).get("vllm") or {}) for plugin in token_producers): + raise ValueError(f"frontend.epp_config sets {TOKEN_PRODUCER} vllm.url, which srtctl points at a worker") topology = config.topology publishing = [ mode @@ -296,14 +316,11 @@ def _validate_kv_events(self, config: Any) -> None: f"frontend.epp_config sets {PRECISE_PREFIX_PRODUCER} kvEventsConfig.{', '.join(sorted(managed))}, " "which srtctl manages: the EPP binds one KV-event socket the workers connect to" ) - token_producers = [plugin for plugin in _plugins(epp_config, TOKEN_PRODUCER) if _vllm_token_producer(plugin)] if not token_producers: raise ValueError( f"{PRECISE_PREFIX_PRODUCER} needs the engine's token IDs: add a {TOKEN_PRODUCER} plugin " "to frontend.epp_config (srtctl points it at a worker's render endpoint)" ) - if any("url" in (_parameters(plugin).get("vllm") or {}) for plugin in token_producers): - raise ValueError(f"frontend.epp_config sets {TOKEN_PRODUCER} vllm.url, which srtctl points at a worker") frontend = config.frontend if ( frontend.enable_multiple_frontends and config.engine_node_count > 1 @@ -314,6 +331,12 @@ def _validate_kv_events(self, config: Any) -> None: ) for mode in publishing: backend = config.backend_for_role(mode) + kv_events = config.roles[mode].kv_events + managed = {"endpoint", "topic"} & kv_events.keys() if isinstance(kv_events, dict) else set() + if managed: + raise ValueError( + f"roles.{mode}.kv_events sets {', '.join(sorted(managed))}, which srtctl manages for llm-d" + ) if backend._is_dp_mode(mode) and not backend.is_external_lb(mode): raise ValueError( f"roles.{mode}.kv_events with data-parallel-size: vLLM publishes every DP rank's events on its " @@ -415,11 +438,14 @@ def render_url(self, backend_processes: list[Process], network_interface: str | process = next((p for p in routable if p.endpoint_mode == "prefill"), routable[0]) return f"http://{self.resolve_worker_host(process.node, network_interface)}:{process.http_port}" - def epp_command(self, config: Any, epp_config_path: str) -> list[str]: - user_args = self.get_frontend_args_list(config.frontend.args) - managed = {str(key).replace("_", "-") for key in (config.frontend.args or {})} & _MANAGED_EPP_FLAGS + def _validate_epp_args(self, args: dict[str, Any] | None) -> None: + managed = {str(key).replace("_", "-") for key in (args or {})} & _MANAGED_EPP_FLAGS if managed: raise ValueError(f"frontend.args sets {', '.join(sorted(managed))}, which srtctl manages for llm-d") + + def epp_command(self, config: Any, epp_config_path: str) -> list[str]: + self._validate_epp_args(config.frontend.args) + user_args = self.get_frontend_args_list(config.frontend.args) return [ *self.executable, f"--pool-name={POOL}", diff --git a/src/srtctl/services/llm_d_sidecar.py b/src/srtctl/services/llm_d_sidecar.py index ce75378fa..0fef160a0 100644 --- a/src/srtctl/services/llm_d_sidecar.py +++ b/src/srtctl/services/llm_d_sidecar.py @@ -39,6 +39,7 @@ # Argv shown by dry-run, where there is no worker to read the ports from. _PREVIEW_PORTS = ["--port=", "--model-server-port="] +_MANAGED_FLAGS = frozenset({"--port", "--model-server-port", "--kv-connector", "--secure-proxy"}) def sidecar_kv_connector(backend: Any) -> str: @@ -74,6 +75,16 @@ class LLMDSidecarService(ServiceKind): def validate(self, service: ServiceConfig, config: SrtConfig) -> None: from srtctl.frontends import FRONTEND_NONE, get_frontend + conflicts = { + arg.split("=", 1)[0] + for arg in [*(service.command or []), *service.args] + if arg.startswith("--") and arg.split("=", 1)[0] in _MANAGED_FLAGS + } + if conflicts: + raise ValidationError( + f"services[{service.name}].command/args sets {', '.join(sorted(conflicts))}, " + "which srtctl manages for the worker's ports and KV-transfer protocol" + ) proxied = ( frozenset() if config.frontend.type == FRONTEND_NONE diff --git a/tests/test_llm_d_frontend.py b/tests/test_llm_d_frontend.py index 24115737e..d3a432158 100644 --- a/tests/test_llm_d_frontend.py +++ b/tests/test_llm_d_frontend.py @@ -19,7 +19,7 @@ from srtctl.core.config import load_config from srtctl.core.schema import SrtConfig from srtctl.core.topology import Endpoint, Process -from srtctl.frontends import LLMDFrontend, get_frontend +from srtctl.frontends import LLMDFrontend, get_frontend, list_frontend_types from srtctl.frontends.llm_d import ( DISCOVERY_PLUGIN, ENDPOINTS_FILE, @@ -150,6 +150,44 @@ def test_declared_sidecar_must_sit_on_proxied_workers() -> None: _load(recipe) +@pytest.mark.parametrize( + "service", + [ + {"name": "llm-d-sidecar", "type": "llm-d-sidecar", "enabled": False}, + {"name": "llm-d-sidecar", "type": "generic", "command": ["sleep", "infinity"]}, + {"name": "extra-sidecar", "type": "llm-d-sidecar"}, + ], +) +def test_pd_requires_exactly_one_enabled_sidecar_service(service: dict) -> None: + recipe = _recipe() + recipe["services"] = [service] + with pytest.raises(ValidationError, match="requires exactly one enabled llm-d-sidecar"): + _load(recipe) + + +def test_sidecar_can_be_renamed_when_the_implied_one_is_disabled() -> None: + recipe = _recipe() + recipe["services"] = [ + {"name": "llm-d-sidecar", "type": "llm-d-sidecar", "enabled": False}, + {"name": "custom-sidecar", "type": "llm-d-sidecar"}, + ] + sidecars = [entry.service for entry in effective_services(_load(recipe)) if entry.service.type == "llm-d-sidecar"] + assert [service.name for service in sidecars] == ["custom-sidecar"] + + +@pytest.mark.parametrize("field", ["command", "args"]) +@pytest.mark.parametrize("flag", ["port", "model-server-port", "kv-connector", "secure-proxy"]) +@pytest.mark.parametrize("equals", [False, True]) +def test_sidecar_rejects_managed_flags_at_config_load(field: str, flag: str, equals: bool) -> None: + recipe = _recipe() + argv = [f"--{flag}=override"] if equals else [f"--{flag}", "override"] + recipe["services"] = [ + {"name": "llm-d-sidecar", "type": "llm-d-sidecar", field: ["pd-sidecar", *argv] if field == "command" else argv} + ] + with pytest.raises(ValidationError, match="which srtctl manages"): + _load(recipe) + + def test_pd_epp_config_needs_scheduling_profiles() -> None: recipe = _recipe() del recipe["frontend"]["epp_config"]["schedulingProfiles"] @@ -157,10 +195,18 @@ def test_pd_epp_config_needs_scheduling_profiles() -> None: _load(recipe) -def test_epp_config_needs_the_llm_d_frontend() -> None: - recipe = _recipe() - recipe["frontend"]["type"] = "vllm-router" - with pytest.raises(ValidationError, match="frontend.epp_config is not supported with frontend.type: vllm-router"): +@pytest.mark.parametrize("frontend_type", [name for name in list_frontend_types() if name != "llm-d"] + ["none"]) +@pytest.mark.parametrize("epp_config", [{}, {"plugins": [{"type": "queue-scorer"}]}]) +def test_epp_config_needs_the_llm_d_frontend(frontend_type: str, epp_config: dict) -> None: + recipe = _recipe(AGG) + recipe["frontend"].update(type=frontend_type, epp_config=epp_config) + if frontend_type == "none": + recipe.pop("roles") + recipe.pop("engine") + recipe["services"] = [{"name": "sleeper", "command": ["sleep", "infinity"], "nodes": 1}] + with pytest.raises( + ValidationError, match=f"frontend.epp_config is not supported with frontend.type: {frontend_type}" + ): _load(recipe) @@ -214,6 +260,22 @@ def test_recipe_cannot_configure_discovery() -> None: _load(recipe) +def test_recipe_cannot_reuse_the_managed_discovery_plugin_name() -> None: + recipe = _recipe(AGG) + recipe["frontend"]["epp_config"]["plugins"].append({"type": "queue-scorer", "name": DISCOVERY_PLUGIN}) + with pytest.raises(ValidationError, match="must not configure endpoint discovery"): + _load(recipe) + + +def test_tokenizer_url_is_managed_even_without_precise_routing() -> None: + recipe = _recipe(AGG) + recipe["frontend"]["epp_config"]["plugins"].append( + {"type": "token-producer", "parameters": {"vllm": {"url": "http://custom-tokenizer:8000"}}} + ) + with pytest.raises(ValidationError, match="sets token-producer vllm.url"): + _load(recipe) + + def test_decode_connector_needs_a_sidecar_protocol() -> None: recipe = _recipe() recipe["engine"]["connector"] = "lmcache" @@ -341,6 +403,27 @@ def test_frontend_args_cannot_move_managed_epp_flags() -> None: LLMDFrontend().epp_command(config, "/logs/c.yaml") +@pytest.mark.parametrize( + "flag", + [ + "pool-name", + "pool-namespace", + "config-file", + "config-text", + "grpc-port", + "grpc-health-port", + "metrics-port", + "secure-serving", + ], +) +@pytest.mark.parametrize("underscores", [False, True]) +def test_managed_epp_flags_are_rejected_at_config_load(flag: str, underscores: bool) -> None: + recipe = _recipe(AGG) + recipe["frontend"]["args"] = {flag.replace("-", "_") if underscores else flag: "override"} + with pytest.raises(ValidationError, match=f"{flag}, which srtctl manages"): + _load(recipe) + + def test_ready_endpoints_gauge_is_parsed() -> None: text = ( "# HELP llm_d_epp_ready_endpoints The number of ready endpoints.\n" @@ -490,6 +573,14 @@ def test_kv_events_need_the_precise_producer_and_vice_versa() -> None: _load(recipe) +@pytest.mark.parametrize("field", ["endpoint", "topic"]) +def test_worker_kv_event_routing_cannot_be_overridden(field: str) -> None: + recipe = _precise(_recipe()) + recipe["roles"]["prefill"]["kv_events"] = {field: "override"} + with pytest.raises(ValidationError, match=f"roles.prefill.kv_events sets {field}, which srtctl manages"): + _load(recipe) + + @pytest.mark.parametrize( ("edit", "message"), [ From 1f0836d2e749bc0e6918135f84499b1d847ca74a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 17:23:07 -0500 Subject: [PATCH 18/25] chore: keep upstream's CODEOWNERS Signed-off-by: Cam Quilici --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 034368670..a05963fe3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,5 +1,5 @@ # Default owners for all files -* @alec-flowers @nlevin-ui @csahithi +* @alec-flowers @ishandhanani @nlevin-ui @csahithi # Curated examples owners /examples/ @kyleliang-nv @hjjq @qiching @kedarpotdar-nv From 28f34b87cc40d48a72036da252616ee389873e27 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 18:00:07 -0500 Subject: [PATCH 19/25] chore: drop the fork's launch-plan recording from the llm-d branch Signed-off-by: Cam Quilici --- docs/cluster-config.md | 2 - docs/config-reference.md | 10 +- docs/schema-reference.md | 4 +- docs/schema/cluster.schema.json | 5 - docs/schema/recipe.schema.json | 7 +- src/srtctl/cli/do_sweep.py | 13 +-- src/srtctl/core/launch_plan.py | 195 -------------------------------- src/srtctl/core/schema.py | 7 +- src/srtctl/core/slurm.py | 12 -- srtslurm.yaml.example | 6 - tests/test_launch_plan.py | 119 ------------------- 11 files changed, 5 insertions(+), 375 deletions(-) delete mode 100644 src/srtctl/core/launch_plan.py delete mode 100644 tests/test_launch_plan.py diff --git a/docs/cluster-config.md b/docs/cluster-config.md index 49fb752ed..ddb137d95 100644 --- a/docs/cluster-config.md +++ b/docs/cluster-config.md @@ -44,8 +44,6 @@ reporting: **output_dir**: When set, job logs are written to `output_dir/{job_id}/logs` instead of `srtctl_root/outputs/{job_id}/logs`. Useful for CI/CD and ephemeral environments. -**record_launch_plan**: Set `true` to record the realized launch plan (`output.record_launch_plan`) for every recipe on the cluster. Because the launch plan lives under the normal log directory, existing S3 postprocessing and external collectors that archive the complete log directory include it without a separate upload path. - **containers**: A map from alias to image path or registry URI. One resolver replaces image aliases in `model.container`, `roles..container`, frontend/benchmark images, exporter images and `services[].container`. Literal paths and registry URIs pass through untouched. Free-form maps (`environment`, `roles..env`, `roles..args`, `services[].env`, `container_mounts`) and the `identity` block are never rewritten. **default_bash_preamble**: A shell snippet (e.g. `"ulimit -n 1048576 -s unlimited -u 1048576"`) prepended to every container srun launched by srtctl: workers, frontends, telemetry, benchmark, postprocess. Runs before per-call `bash_preamble` and the main command, so cluster-wide ulimits apply to everything downstream. Silently dropped for distroless containers (e.g. `prom/node-exporter`) that bypass the bash wrapper; a WARNING log is emitted in that case. diff --git a/docs/config-reference.md b/docs/config-reference.md index 4b169b265..95c63bb5d 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -55,23 +55,15 @@ Fields: [ModelConfig](schema-reference.md#modelconfig). ## output -Output paths and optional reproducibility artifacts. +Output configuration with formattable paths. ```yaml output: log_dir: "./outputs/{job_id}/logs" - record_launch_plan: false # Save exact realized srun scripts and a manifest ``` The `log_dir` supports FormattablePath templating. See [FormattablePath Template System](runtime-env.md#formattablepath-template-system). -When `record_launch_plan` is enabled, srtctl creates `logs/launch-plan/manifest.json` and one executable shell -script for every realized `srun` invocation. Recording happens after Slurm assigns nodes, ports, heterogeneous -groups, mounts, and container paths, so the scripts describe what was actually launched rather than a pre-submit -estimate. Secret-like environment values are never persisted; the scripts name the environment variables that -must be supplied for replay. The resolved recipe, outer `sbatch_script.sh`, lockfile, and resource snapshot remain -alongside this directory and are referenced by the manifest. - ## health_check Health check configuration for worker readiness, and the worker log watch that fails a run early. diff --git a/docs/schema-reference.md b/docs/schema-reference.md index cc926a355..0e13be02a 100644 --- a/docs/schema-reference.md +++ b/docs/schema-reference.md @@ -249,12 +249,11 @@ Profiling configuration. ### OutputConfig -Output paths and optional reproducibility artifacts. +Output configuration with formattable paths. | Key | Type | Default | Description | |---|---|---|---| | `log_dir` | [FormattablePath](#formattablepath) | `()` | Directory for job logs and results; a FormattablePath, so `{job_id}` and `$VARS` expand. | -| `record_launch_plan` | bool | `False` | Save the realized `srun` scripts and a manifest under `logs/launch-plan/`. | ### HealthCheckConfig @@ -801,7 +800,6 @@ Top-level keys of `srtslurm.yaml`. Recipes inherit these defaults and resolve al | `default_health_check` | dict[str, int] \| None | `None` | `health_check` block (`max_attempts`, `interval_seconds`) used when a recipe has none. | | `srtctl_root` | str \| None | `None` | srtctl checkout on the shared filesystem that compute nodes mount at /srtctl-src. Default: this checkout. | | `output_dir` | str \| None | `None` | Custom output directory for job logs | -| `record_launch_plan` | bool | `False` | Cluster-wide default for recording exact realized srun commands. Recipes can opt in independently with output.record_launch_plan. | | `model_paths` | dict[str, str] \| None | `None` | Alias -> path map; a recipe's `model.path` may name an alias instead of a path. | | `containers` | dict[str, str] \| None | `None` | Alias -> image map, resolved for every container key in a recipe (`model.container`, `roles..container`, ...). | | `cloud` | dict[str, str] \| None | `None` | Free-form cloud settings. Accepted for compatibility; srtctl does not read it. | diff --git a/docs/schema/cluster.schema.json b/docs/schema/cluster.schema.json index cffd4cc13..3b3f01d95 100644 --- a/docs/schema/cluster.schema.json +++ b/docs/schema/cluster.schema.json @@ -178,11 +178,6 @@ "description": "Custom output directory for job logs", "default": null }, - "record_launch_plan": { - "type": "boolean", - "description": "Cluster-wide default for recording exact realized srun commands. Recipes can opt in independently with output.record_launch_plan.", - "default": false - }, "model_paths": { "anyOf": [ { diff --git a/docs/schema/recipe.schema.json b/docs/schema/recipe.schema.json index e6cd20cb8..0f0044165 100644 --- a/docs/schema/recipe.schema.json +++ b/docs/schema/recipe.schema.json @@ -1559,14 +1559,9 @@ "log_dir": { "type": "string", "description": "Directory for job logs and results; a FormattablePath, so `{job_id}` and `$VARS` expand." - }, - "record_launch_plan": { - "type": "boolean", - "description": "Save the realized `srun` scripts and a manifest under `logs/launch-plan/`.", - "default": false } }, - "description": "Output paths and optional reproducibility artifacts." + "description": "Output configuration with formattable paths." }, "PlacementConfig": { "type": "object", diff --git a/src/srtctl/cli/do_sweep.py b/src/srtctl/cli/do_sweep.py index 89bfdd7f3..73ea2a7da 100644 --- a/src/srtctl/cli/do_sweep.py +++ b/src/srtctl/cli/do_sweep.py @@ -33,9 +33,8 @@ TelemetryStageMixin, WorkerStageMixin, ) -from srtctl.core.config import get_srtslurm_setting, load_config +from srtctl.core.config import load_config from srtctl.core.health import wait_for_port -from srtctl.core.launch_plan import configure_launch_plan from srtctl.core.lockfile import write_lockfile from srtctl.core.processes import ( ProcessRegistry, @@ -709,12 +708,6 @@ def _run_post_eval(self, stop_event: threading.Event) -> int: def run(self) -> int: """Run the complete sweep.""" - record_launch_plan = self.config.output.record_launch_plan or bool( - get_srtslurm_setting("record_launch_plan", False) - ) - launch_plan_dir = self.runtime.log_dir / "launch-plan" if record_launch_plan else None - configure_launch_plan(launch_plan_dir, job_id=self.runtime.job_id) - logger.info("Sweep Orchestrator") logger.info("Job ID: %s", self.runtime.job_id) logger.info("Run name: %s", self.runtime.run_name) @@ -726,10 +719,6 @@ def run(self) -> int: logger.info("Pool %s: %s", pool, ", ".join(nodes)) if self.config.profiling.enabled: logger.info("Profiling: %s", self.config.profiling.type) - if launch_plan_dir is not None: - logger.info("Recording realized Slurm launch plan: %s", launch_plan_dir) - # Preserve submission provenance before any runtime stage can fail. - self._copy_config_to_logs() resource_snapshot = record_resource_snapshot(self.config, self.runtime) diff --git a/src/srtctl/core/launch_plan.py b/src/srtctl/core/launch_plan.py deleted file mode 100644 index 8e3d3e85d..000000000 --- a/src/srtctl/core/launch_plan.py +++ /dev/null @@ -1,195 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Persist the realized Slurm launch plan for post-run reproduction.""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import shlex -import threading -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -_SECRET_NAME = re.compile(r"(?:^|_)(?:TOKEN|SECRET|PASSWORD|PASSWD|API_KEY|ACCESS_KEY|PRIVATE_KEY|CREDENTIAL)(?:_|$)") -_SAFE_NAME = re.compile(r"[^A-Za-z0-9_.-]+") - - -def _utc_now() -> str: - return datetime.now(timezone.utc).isoformat() - - -def _is_secret_name(name: str) -> bool: - return bool(_SECRET_NAME.search(name.upper())) - - -def _slug(value: str) -> str: - normalized = _SAFE_NAME.sub("-", value).strip("-.").lower() - return normalized[:80] or "step" - - -def _redacted_command( - command: list[str], - *, - env_to_set: dict[str, str] | None, - srun_export_env: dict[str, str] | None, -) -> tuple[list[str], list[str]]: - """Return a replayable command with secret environment values removed.""" - replay = list(command) - required_secrets: set[str] = set() - - if srun_export_env: - for idx, arg in enumerate(replay): - if not arg.startswith("--export=ALL,"): - continue - exports = [] - for item in arg.removeprefix("--export=ALL,").split(","): - name, separator, value = item.partition("=") - if separator and _is_secret_name(name): - exports.append(name) - required_secrets.add(name) - else: - exports.append(item) - replay[idx] = "--export=ALL," + ",".join(exports) - - if env_to_set: - bash_index = next((idx for idx in range(len(replay) - 1) if replay[idx : idx + 2] == ["bash", "-c"]), None) - if bash_index is not None: - bash_command = replay[bash_index + 2] - for name, value in env_to_set.items(): - if not _is_secret_name(name): - continue - actual = f"export {name}={shlex.quote(value)}" - replacement = f'export {name}="${{{name}:?Set {name} to replay this step}}"' - bash_command = bash_command.replace(actual, replacement) - required_secrets.add(name) - replay[bash_index + 2] = bash_command - - return replay, sorted(required_secrets) - - -class LaunchPlanRecorder: - """Thread-safe recorder for exact, realized ``srun`` invocations.""" - - def __init__(self, directory: Path, *, job_id: str | None = None) -> None: - self.directory = directory - self.directory.mkdir(parents=True, exist_ok=True) - self.job_id = job_id or os.environ.get("SLURM_JOB_ID") or os.environ.get("SLURM_JOBID") - self._lock = threading.Lock() - self._entries: list[dict[str, Any]] = [] - self._write_manifest() - - def record( - self, - command: list[str], - *, - label: str, - output: str | None, - nodelist: list[str] | None, - het_group: int | None, - container_image: str | None, - env_to_set: dict[str, str] | None, - srun_export_env: dict[str, str] | None, - ) -> Path: - replay, required_secrets = _redacted_command( - command, - env_to_set=env_to_set, - srun_export_env=srun_export_env, - ) - with self._lock: - sequence = len(self._entries) + 1 - filename = f"{sequence:03d}-{_slug(label)}.sh" - script_path = self.directory / filename - rendered = self._render_script(replay, required_secrets=required_secrets) - script_path.write_text(rendered) - script_path.chmod(0o750) - - entry = { - "sequence": sequence, - "script": filename, - "recorded_at": _utc_now(), - "output": output, - "nodelist": list(nodelist or []), - "het_group": het_group, - "container_image": container_image, - "required_secret_environment": required_secrets, - "sha256": hashlib.sha256(rendered.encode()).hexdigest(), - } - self._entries.append(entry) - self._write_manifest() - return script_path - - def _render_script(self, command: list[str], *, required_secrets: list[str]) -> str: - lines = [ - "#!/usr/bin/env bash", - "set -euo pipefail", - "", - "# Generated from the exact srun argv used by srt-slurm.", - "# Run inside a compatible active Slurm allocation with the recorded recipe and container available.", - ] - if required_secrets: - lines.append("# Required secret environment (values intentionally omitted): " + ", ".join(required_secrets)) - for name in required_secrets: - lines.append(f': "${{{name}:?Set {name} to replay this step}}"') - lines.extend(["", "exec " + shlex.join(command), ""]) - return "\n".join(lines) - - def _write_manifest(self) -> None: - manifest = { - "schema_version": 1, - "job_id": self.job_id, - "slurm_nodelist": os.environ.get("SLURM_NODELIST"), - "generated_at": _utc_now(), - "replay_scope": "Individual realized Slurm steps; orchestration order and lifecycle remain owned by srtctl.", - "secret_policy": "Secret-like environment values are omitted and named in required_secret_environment.", - "related_artifacts": [ - "../config.yaml", - "../sbatch_script.sh", - "../recipe.lock.yaml", - "../resource_snapshot.json", - ], - "steps": self._entries, - } - temporary = self.directory / ".manifest.json.tmp" - temporary.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") - temporary.replace(self.directory / "manifest.json") - - -_active_recorder: LaunchPlanRecorder | None = None - - -def configure_launch_plan(directory: Path | None, *, job_id: str | None = None) -> LaunchPlanRecorder | None: - """Enable or disable process-wide launch recording for this orchestrator.""" - global _active_recorder - _active_recorder = LaunchPlanRecorder(directory, job_id=job_id) if directory is not None else None - return _active_recorder - - -def record_srun_command( - command: list[str], - *, - label: str, - output: str | None, - nodelist: list[str] | None, - het_group: int | None, - container_image: str | None, - env_to_set: dict[str, str] | None, - srun_export_env: dict[str, str] | None, -) -> Path | None: - """Record a realized command when launch-plan capture is enabled.""" - if _active_recorder is None: - return None - return _active_recorder.record( - command, - label=label, - output=output, - nodelist=nodelist, - het_group=het_group, - container_image=container_image, - env_to_set=env_to_set, - srun_export_env=srun_export_env, - ) diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 352755f2f..f4f4f2af7 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -411,9 +411,6 @@ class ClusterConfig: # srtctl checkout on the shared filesystem that compute nodes mount at /srtctl-src. Default: this checkout. srtctl_root: str | None = None output_dir: str | None = None # Custom output directory for job logs - # Cluster-wide default for recording exact realized srun commands. Recipes - # can opt in independently with output.record_launch_plan. - record_launch_plan: bool = False # Alias -> path map; a recipe's `model.path` may name an alias instead of a path. model_paths: dict[str, str] | None = None # Alias -> image map, resolved for every container key in a recipe (`model.container`, `roles..container`, ...). @@ -2611,14 +2608,12 @@ class FrontendConfig: @dataclass(frozen=True) class OutputConfig: - """Output paths and optional reproducibility artifacts.""" + """Output configuration with formattable paths.""" # Directory for job logs and results; a FormattablePath, so `{job_id}` and `$VARS` expand. log_dir: Annotated[FormattablePath, FormattablePathField()] = field( default_factory=lambda: FormattablePath(template="./outputs/{job_id}/logs") ) - # Save the realized `srun` scripts and a manifest under `logs/launch-plan/`. - record_launch_plan: bool = False Schema: ClassVar[type[Schema]] = Schema diff --git a/src/srtctl/core/slurm.py b/src/srtctl/core/slurm.py index 20724e4a2..1aaa4665e 100644 --- a/src/srtctl/core/slurm.py +++ b/src/srtctl/core/slurm.py @@ -20,7 +20,6 @@ from pathlib import Path from .ip_utils import get_node_ip -from .launch_plan import record_srun_command logger = logging.getLogger(__name__) @@ -367,17 +366,6 @@ def start_srun_process( # the srtctl logger to DEBUG when troubleshooting srun arg construction. logger.debug("srun command: %s", shlex.join(srun_cmd)) - record_srun_command( - srun_cmd, - label=Path(output).stem if output else Path(command[0]).name, - output=output, - nodelist=list(nodelist) if nodelist else None, - het_group=het_group, - container_image=str(container_image) if container_image else None, - env_to_set=env_to_set, - srun_export_env=srun_export_env, - ) - # Start the process proc = subprocess.Popen( srun_cmd, diff --git a/srtslurm.yaml.example b/srtslurm.yaml.example index cb305a719..932131f3c 100644 --- a/srtslurm.yaml.example +++ b/srtslurm.yaml.example @@ -76,12 +76,6 @@ use_segment_sbatch_directive: true # Default: true # Set to true if your cluster requires --exclusive to be set. use_exclusive_sbatch_directive: false # Default: false -# Reproducibility artifacts (optional) -# Record every realized srun invocation as an executable, secret-redacted script -# under outputs//logs/launch-plan/. Individual recipes can also enable -# this with output.record_launch_plan: true. -record_launch_plan: false - # Container aliases (optional; a recipe may also name the image directly in model.container) containers: sglang-latest: "/shared/containers/sglang-v0.4.sqsh" diff --git a/tests/test_launch_plan.py b/tests/test_launch_plan.py deleted file mode 100644 index 24989f137..000000000 --- a/tests/test_launch_plan.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for opt-in recording of realized Slurm launch commands.""" - -import json -import subprocess -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -from srtctl.core.launch_plan import configure_launch_plan -from srtctl.core.schema import ClusterConfig, OutputConfig -from srtctl.core.slurm import start_srun_process - - -@pytest.fixture(autouse=True) -def _reset_launch_plan(): - configure_launch_plan(None) - yield - configure_launch_plan(None) - - -def _launch(tmp_path: Path, **kwargs) -> Path: - plan_dir = tmp_path / "logs" / "launch-plan" - configure_launch_plan(plan_dir, job_id="42042") - with ( - patch("srtctl.core.slurm.get_slurm_job_id", return_value="42042"), - patch("srtctl.core.slurm._get_cluster_bash_preamble", return_value=None), - patch("subprocess.Popen", return_value=MagicMock()), - ): - start_srun_process(["python3", "-m", "worker"], **kwargs) - return plan_dir - - -def test_launch_plan_is_opt_in_for_recipe_and_cluster() -> None: - assert OutputConfig().record_launch_plan is False - assert OutputConfig(record_launch_plan=True).record_launch_plan is True - assert ClusterConfig().record_launch_plan is False - assert ClusterConfig(record_launch_plan=True).record_launch_plan is True - - -def test_disabled_recorder_writes_nothing(tmp_path: Path) -> None: - with ( - patch("srtctl.core.slurm.get_slurm_job_id", return_value="42042"), - patch("srtctl.core.slurm._get_cluster_bash_preamble", return_value=None), - patch("subprocess.Popen", return_value=MagicMock()), - ): - start_srun_process(["python3", "-m", "worker"]) - - assert not (tmp_path / "launch-plan").exists() - - -def test_records_exact_realized_srun_as_executable_script(tmp_path: Path) -> None: - output = tmp_path / "logs" / "node-a_prefill_w0.out" - plan_dir = _launch( - tmp_path, - nodelist=["node-a"], - output=str(output), - container_image="/containers/vllm.sqsh", - container_mounts={Path("/models/model"): Path("/model")}, - env_to_set={"NCCL_DEBUG": "INFO"}, - het_group=0, - ) - - scripts = list(plan_dir.glob("*.sh")) - assert [path.name for path in scripts] == ["001-node-a_prefill_w0.sh"] - script = scripts[0].read_text() - assert scripts[0].stat().st_mode & 0o100 - assert "exec srun --jobid 42042 --overlap" in script - assert "--nodelist node-a" in script - assert "--het-group=0" in script - assert "--container-image /containers/vllm.sqsh" in script - assert "export NCCL_DEBUG=INFO" in script - assert "python3 -m worker" in script - syntax = subprocess.run(["bash", "-n", str(scripts[0])], capture_output=True, text=True, check=False) - assert syntax.returncode == 0, syntax.stderr - - manifest = json.loads((plan_dir / "manifest.json").read_text()) - assert manifest["job_id"] == "42042" - assert manifest["steps"][0]["script"] == scripts[0].name - assert manifest["steps"][0]["nodelist"] == ["node-a"] - assert manifest["steps"][0]["container_image"] == "/containers/vllm.sqsh" - - -def test_secret_environment_values_are_not_persisted(tmp_path: Path) -> None: - plan_dir = _launch( - tmp_path, - output=str(tmp_path / "logs" / "benchmark.out"), - env_to_set={"HF_TOKEN": "hf-super-secret", "PUBLIC_VALUE": "visible"}, - srun_export_env={"AWS_SECRET_ACCESS_KEY": "aws-super-secret", "ENROOT_REMAP_ROOT": "yes"}, - ) - - script = next(plan_dir.glob("*.sh")).read_text() - manifest_text = (plan_dir / "manifest.json").read_text() - combined = script + manifest_text - assert "hf-super-secret" not in combined - assert "aws-super-secret" not in combined - assert 'export HF_TOKEN="${HF_TOKEN:?Set HF_TOKEN to replay this step}"' in script - assert "--export=ALL,AWS_SECRET_ACCESS_KEY,ENROOT_REMAP_ROOT=yes" in script - assert "export PUBLIC_VALUE=visible" in script - - manifest = json.loads(manifest_text) - assert manifest["steps"][0]["required_secret_environment"] == ["AWS_SECRET_ACCESS_KEY", "HF_TOKEN"] - - -def test_multiple_commands_have_stable_order_and_distinct_files(tmp_path: Path) -> None: - plan_dir = _launch(tmp_path, output=str(tmp_path / "logs" / "infra.out")) - with ( - patch("srtctl.core.slurm.get_slurm_job_id", return_value="42042"), - patch("srtctl.core.slurm._get_cluster_bash_preamble", return_value=None), - patch("subprocess.Popen", return_value=MagicMock()), - ): - start_srun_process(["python3", "bench.py"], output=str(tmp_path / "logs" / "benchmark.out")) - - manifest = json.loads((plan_dir / "manifest.json").read_text()) - assert [entry["sequence"] for entry in manifest["steps"]] == [1, 2] - assert [entry["script"] for entry in manifest["steps"]] == ["001-infra.sh", "002-benchmark.sh"] From e475df04bdce360ade300712aca2477a46fac502 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 18:03:39 -0500 Subject: [PATCH 20/25] refactor(vllm): expose DP mode to frontends --- src/srtctl/backends/vllm.py | 16 ++++++++-------- src/srtctl/frontends/llm_d.py | 2 +- src/srtctl/frontends/vllm_router.py | 2 +- tests/test_configs.py | 8 ++++---- tests/test_vllm_router_frontend.py | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index cb60a0c5c..11ae121e8 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -688,7 +688,7 @@ def allocate_endpoints( ), ) - def _is_dp_mode(self, mode: WorkerMode) -> bool: + def is_dp_mode(self, mode: WorkerMode) -> bool: """Check if this mode uses Data Parallel + Expert Parallel pattern. DP+EP mode is detected when data-parallel-size is set in the mode's config. @@ -699,7 +699,7 @@ def _is_dp_mode(self, mode: WorkerMode) -> bool: def is_external_lb(self, mode: WorkerMode) -> bool: """Whether ``data-parallel-external-lb`` exposes each DP rank as a separate HTTP server.""" - return self._is_dp_mode(mode) and any( + return self.is_dp_mode(mode) and any( normalize_vllm_config_key(key) == "data-parallel-external-lb" and value for key, value in self.get_config_for_mode(mode).items() ) @@ -881,7 +881,7 @@ def endpoints_to_processes( [endpoint], port_allocator=allocator, sidecar_grpc=dynamo_sidecar, bootstrap_ports=False ) ) - elif not any(self._is_dp_mode(ep.mode) for ep in endpoints): + elif not any(self.is_dp_mode(ep.mode) for ep in endpoints): # Standard TP mode: one process per node, or one per engine of the # worker under backend.failover (engine 0 plus its shadows). processes = endpoints_to_processes( @@ -934,7 +934,7 @@ def _dp_per_gpu_endpoints_to_processes( processes: list[Process] = [] for endpoint in endpoints: - if not self._is_dp_mode(endpoint.mode): + if not self.is_dp_mode(endpoint.mode): # Non-DP endpoints get standard processing (all modes are normally consistent). for node_rank, node in enumerate(endpoint.nodes): is_leader = node_rank == 0 @@ -1003,7 +1003,7 @@ def _dp_per_node_endpoints_to_processes( processes: list[Process] = [] for endpoint in endpoints: - if not self._is_dp_mode(endpoint.mode): + if not self.is_dp_mode(endpoint.mode): processes.extend( endpoints_to_processes( [endpoint], port_allocator=allocator, sidecar_grpc=sidecar_grpc, bootstrap_ports=False @@ -1176,7 +1176,7 @@ def build_worker_command( # Collected as the command is built so the override report below can # name the value srtslurm actually passed for each flag it took over. srtslurm_owned: dict[str, str] = {} - is_dp_mode = self._is_dp_mode(mode) + is_dp_mode = self.is_dp_mode(mode) replica_size = self._get_model_parallel_size(mode) local_gpu_count = len(process.gpu_indices) spans_nodes = replica_size > local_gpu_count @@ -1338,7 +1338,7 @@ def build_worker_command( cmd.extend(["--device-ids", device_ids]) # Check if this is DP+EP mode (data-parallel-size set) - is_dp_mode = self._is_dp_mode(mode) + is_dp_mode = self.is_dp_mode(mode) if is_dp_mode and self.dp_launch_mode == "per_node": rpc_port_kebab = config.pop("data-parallel-rpc-port", None) rpc_port_snake = config.pop("data_parallel_rpc_port", None) @@ -1474,7 +1474,7 @@ def _build_sidecar_command( ) -> list[str]: """Expose local DP frontends or one frontend for a cross-node replica.""" mode = process.endpoint_mode - is_dp_mode = self._is_dp_mode(mode) + is_dp_mode = self.is_dp_mode(mode) endpoint_nodes = list(dict.fromkeys(candidate.node for candidate in endpoint_processes)) is_multi_node = len(endpoint_nodes) > 1 multi_node_replica = is_multi_node and ( diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py index 577214e08..d1ef582a6 100644 --- a/src/srtctl/frontends/llm_d.py +++ b/src/srtctl/frontends/llm_d.py @@ -337,7 +337,7 @@ def _validate_kv_events(self, config: Any) -> None: raise ValueError( f"roles.{mode}.kv_events sets {', '.join(sorted(managed))}, which srtctl manages for llm-d" ) - if backend._is_dp_mode(mode) and not backend.is_external_lb(mode): + if backend.is_dp_mode(mode) and not backend.is_external_lb(mode): raise ValueError( f"roles.{mode}.kv_events with data-parallel-size: vLLM publishes every DP rank's events on its " "own port, so the EPP must route to the ranks themselves; set " diff --git a/src/srtctl/frontends/vllm_router.py b/src/srtctl/frontends/vllm_router.py index e428943aa..8f5d77eca 100644 --- a/src/srtctl/frontends/vllm_router.py +++ b/src/srtctl/frontends/vllm_router.py @@ -92,7 +92,7 @@ def validate(self, config: Any) -> None: if gpu_count <= 0: continue backend = config.backend_for_role(mode) - if not backend._is_dp_mode(mode): + if not backend.is_dp_mode(mode): expansion_by_mode[mode] = 1 continue if backend.dp_launch_mode != "per_node": diff --git a/tests/test_configs.py b/tests/test_configs.py index f4dd68dce..0eb9d610e 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -2599,8 +2599,8 @@ def test_dp_mode_detection(self): "decode": RoleConfig(args={"tensor-parallel-size": 4}), } ) - assert backend._is_dp_mode("prefill") is False - assert backend._is_dp_mode("decode") is False + assert backend.is_dp_mode("prefill") is False + assert backend.is_dp_mode("decode") is False # DP mode detected when data-parallel-size is set backend_dp = VLLMBackend( @@ -2609,8 +2609,8 @@ def test_dp_mode_detection(self): "decode": RoleConfig(args={"data-parallel-size": 16, "enable-expert-parallel": True}), } ) - assert backend_dp._is_dp_mode("prefill") is True - assert backend_dp._is_dp_mode("decode") is True + assert backend_dp.is_dp_mode("prefill") is True + assert backend_dp.is_dp_mode("decode") is True assert backend_dp._get_dp_size("prefill") == 16 def test_dp_per_gpu_mode_creates_per_gpu_processes(self): diff --git a/tests/test_vllm_router_frontend.py b/tests/test_vllm_router_frontend.py index efa4bef22..24019ba62 100644 --- a/tests/test_vllm_router_frontend.py +++ b/tests/test_vllm_router_frontend.py @@ -426,7 +426,7 @@ def test_dp_size_one_is_not_a_distributed_launch_mode() -> None: backend = VLLMBackend(roles={"agg": RoleConfig(args={"data-parallel-size": 1})}) assert backend.find_dp_modes() == [] - assert backend._is_dp_mode("agg") is False + assert backend.is_dp_mode("agg") is False def test_router_validates_pcp_as_part_of_the_vllm_world_size() -> None: From f33d6ee1398cb19a68c947cce282e38abecb9e59 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 18:03:46 -0500 Subject: [PATCH 21/25] refactor: read typed process DP ranks directly --- src/srtctl/cli/mixins/worker_stage.py | 2 +- src/srtctl/core/supervisor.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/srtctl/cli/mixins/worker_stage.py b/src/srtctl/cli/mixins/worker_stage.py index 983602f71..6c32c1545 100644 --- a/src/srtctl/cli/mixins/worker_stage.py +++ b/src/srtctl/cli/mixins/worker_stage.py @@ -436,7 +436,7 @@ def __missing__(self, key: str) -> str: env_to_unset = ["VLLM_PORT"] if backend.type == "vllm" and len(endpoint_nodes) > 1 else None step_name = worker_step_name( - mode, index, process.node, attempt, getattr(process, "engine_id", 0), getattr(process, "dp_rank", None) + mode, index, process.node, attempt, getattr(process, "engine_id", 0), process.dp_rank ) proc = start_srun_process( command=cmd, diff --git a/src/srtctl/core/supervisor.py b/src/srtctl/core/supervisor.py index 3fea8078e..2730c48d0 100644 --- a/src/srtctl/core/supervisor.py +++ b/src/srtctl/core/supervisor.py @@ -191,9 +191,7 @@ def track( if not policy.enabled: continue expected = { - worker_step_name( - mode, index, process.node, engine_id=engine, dp_rank=getattr(process, "dp_rank", None) - ) + worker_step_name(mode, index, process.node, engine_id=engine, dp_rank=process.dp_rank) for process in processes } mine = sorted(names & expected) From bdd7ddd88a851544d3ecbc1f71cc3f3289cd4522 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 18:04:01 -0500 Subject: [PATCH 22/25] refactor(llm-d): validate EPP arguments only at config load --- src/srtctl/frontends/llm_d.py | 1 - tests/test_llm_d_frontend.py | 6 ------ 2 files changed, 7 deletions(-) diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py index d1ef582a6..09188f759 100644 --- a/src/srtctl/frontends/llm_d.py +++ b/src/srtctl/frontends/llm_d.py @@ -444,7 +444,6 @@ def _validate_epp_args(self, args: dict[str, Any] | None) -> None: raise ValueError(f"frontend.args sets {', '.join(sorted(managed))}, which srtctl manages for llm-d") def epp_command(self, config: Any, epp_config_path: str) -> list[str]: - self._validate_epp_args(config.frontend.args) user_args = self.get_frontend_args_list(config.frontend.args) return [ *self.executable, diff --git a/tests/test_llm_d_frontend.py b/tests/test_llm_d_frontend.py index d3a432158..aede3fa31 100644 --- a/tests/test_llm_d_frontend.py +++ b/tests/test_llm_d_frontend.py @@ -397,12 +397,6 @@ def test_start_frontends_writes_the_router_files_and_starts_epp_then_envoy(tmp_p assert envoy_cmd == ["envoy", "-c", "/logs/llm-d-envoy.yaml", "--disable-hot-restart"] -def test_frontend_args_cannot_move_managed_epp_flags() -> None: - config = SimpleNamespace(frontend=SimpleNamespace(args={"metrics_port": 1234})) - with pytest.raises(ValueError, match="metrics-port, which srtctl manages"): - LLMDFrontend().epp_command(config, "/logs/c.yaml") - - @pytest.mark.parametrize( "flag", [ From 572a84440ae16aedf72c421b61ce7f4404a2283c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 8 Oct 2026 18:05:06 -0500 Subject: [PATCH 23/25] refactor(llm-d): centralize and document KV event port offsets --- docs/llm-d.md | 14 ++++- src/srtctl/backends/vllm.py | 10 +++- src/srtctl/ports.py | 9 +++ tests/test_design_rule_port_arithmetic.py | 64 -------------------- tests/test_design_rules.py | 72 ----------------------- 5 files changed, 29 insertions(+), 140 deletions(-) delete mode 100644 tests/test_design_rule_port_arithmetic.py diff --git a/docs/llm-d.md b/docs/llm-d.md index 9bf1ae223..c5d74813e 100644 --- a/docs/llm-d.md +++ b/docs/llm-d.md @@ -226,11 +226,21 @@ srtctl wires the rest: endpoint the EPP routes to (a decode worker's sidecar port) and the model requests name; the producer files the events under that endpoint ([`vllm_adapter.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/kvevents/engineadapter/vllm_adapter.go#L52-L58)). - vLLM adds a publisher's DP rank to its port, so an external-LB rank `r` is given - `5557 - r`. + vLLM [adds the publisher's DP rank to its port](https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/distributed/kv_events.py#L336-L340), + so an external-LB rank `r` is given `5557 - r`. The engine sets + [`data_parallel_index` to the global rank](https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/v1/engine/core.py#L1303-L1313), + which the [scheduler passes to the publisher](https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/v1/core/sched/scheduler.py#L161-L164). + An explicit TCP address [connects rather than binds](https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/distributed/kv_events.py#L410-L427), + so all ranks reach the EPP's one subscriber socket. - The token producer's `vllm.url` is the first prefill worker's (else the first worker's) own vLLM API, and `modelName` defaults to the served model name. +These vLLM references are pinned to `ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9`, +shipped in the tested `quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly` image. +The [run artifact](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37396765122/artifacts/11388917384) +records `VLLM_IMAGE_TAG=vllm/vllm-openai:nightly-ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9` +and installed version `0.27.2rc1.dev77+gac7509e2b` in `fingerprint_decode_w0.json`. + With vLLM's `OffloadingConnector` and its `self_describing_kv_events: true`, a worker also reports the blocks it offloads to host memory; the producer indexes them as the `cpu` tier and weights tiers by `indexerConfig.kvCacheBackendConfigs` (`gpu` 1.0, `cpu` 0.8 by diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index 11ae121e8..77839b09e 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -47,6 +47,7 @@ VLLM_MASTER_PORT_BASE, VLLM_MASTER_PORT_STRIDE, VLLM_SCAN_PORTS, + rank_offset_subscriber_port, ) if TYPE_CHECKING: @@ -1297,9 +1298,14 @@ def build_worker_command( subscriber = frontend.kv_events_subscriber(process, runtime, served_model_name) if kv_events else None if kv_events and subscriber is not None: host, port, topic = subscriber - # Offset vLLM's rank addition so every publisher reaches the same subscriber port. + # Tested vLLM uses the global DP rank for the publisher offset: + # https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/v1/engine/core.py#L1303-L1313 + # https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/v1/core/sched/scheduler.py#L161-L164 + # Explicit TCP addresses connect rather than bind: + # https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/distributed/kv_events.py#L410-L427 rank = process.dp_rank or 0 - kv_events = {**kv_events, "endpoint": f"tcp://{host}:{port - rank}", "topic": topic} + publisher_port = rank_offset_subscriber_port(port, rank) + kv_events = {**kv_events, "endpoint": f"tcp://{host}:{publisher_port}", "topic": topic} cmd.extend(["--kv-events-config", json.dumps(kv_events)]) cmd.extend(_config_to_cli_args(config)) return cmd diff --git a/src/srtctl/ports.py b/src/srtctl/ports.py index acf1ac57d..cbfde1a48 100644 --- a/src/srtctl/ports.py +++ b/src/srtctl/ports.py @@ -106,6 +106,15 @@ DYNAMO_SIDECAR_GRPC_PORT = 50051 +def rank_offset_subscriber_port(port: int, rank: int) -> int: + """Undo vLLM's global DP-rank offset to reach a shared KV-event subscriber. + + Tested image's offset_endpoint_port: + https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/distributed/kv_events.py#L495-L523 + """ + return port - rank + + @dataclass(frozen=True) class PortKind: """One kind of listener a worker process binds, allocated by ``NodePortAllocator``. diff --git a/tests/test_design_rule_port_arithmetic.py b/tests/test_design_rule_port_arithmetic.py deleted file mode 100644 index 87c53c270..000000000 --- a/tests/test_design_rule_port_arithmetic.py +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Outbound subscriber offsets must not hide derived listener ports.""" - -import ast -import textwrap - -import pytest - -from tests.test_design_rules import port_arithmetic - - -def _violations(body: str) -> set[str]: - tree = ast.parse("def configure(frontend, process, runtime, model, rank):\n" + textwrap.indent(body, " ")) - return {site for _, site in port_arithmetic("backends/example.py", tree)} - - -@pytest.mark.parametrize( - "holder", ["", "target = frontend.kv_events_subscriber(process, runtime, model) if rank else None\n"] -) -@pytest.mark.parametrize("operation", ["+", "-"]) -def test_subscriber_destination_offset_is_not_a_listener(holder: str, operation: str) -> None: - source = "target" if holder else "frontend.kv_events_subscriber(process, runtime, model)" - assert not _violations( - holder - + f"remote_host, remote_port, topic = {source}\n" - + f'config = {{"endpoint": f"tcp://{{remote_host}}:{{remote_port {operation} rank}}"}}\n' - ) - - -@pytest.mark.parametrize( - "body", - [ - "socket.bind((remote_host, remote_port - rank))", - "listener_port = remote_port + 1", - 'config = {"endpoint": f"tcp://*:{remote_port - rank}"}', - 'config = {"endpoint": f"tcp://{local_host}:{remote_port - rank}"}', - 'config = {"listen": f"tcp://{remote_host}:{remote_port - rank}"}', - 'remote_host = "0.0.0.0"\nconfig = {"endpoint": f"tcp://{remote_host}:{remote_port - rank}"}', - 'remote_port = 8000\nconfig = {"endpoint": f"tcp://{remote_host}:{remote_port - rank}"}', - ], -) -def test_subscriber_port_does_not_exempt_listener_or_reassigned_address(body: str) -> None: - assert _violations( - "remote_host, remote_port, topic = frontend.kv_events_subscriber(process, runtime, model)\n" + body - ) - - -def test_unknown_endpoint_source_is_still_checked() -> None: - assert _violations( - 'host, port, topic = frontend.worker_address(process)\nconfig = {"endpoint": f"tcp://{host}:{port - rank}"}\n' - ) == {"port - rank"} - - -def test_subscriber_binding_does_not_leak_into_another_function() -> None: - tree = ast.parse( - "def publisher(frontend):\n" - " host, port, topic = frontend.kv_events_subscriber()\n" - ' config = {"endpoint": f"tcp://{host}:{port - rank}"}\n' - "def listener(host, port, rank):\n" - ' config = {"endpoint": f"tcp://{host}:{port - rank}"}\n' - ) - assert list(port_arithmetic("backends/example.py", tree)) == [("backends/example.py", "port - rank")] diff --git a/tests/test_design_rules.py b/tests/test_design_rules.py index 6ca88a729..dbcc2fb43 100644 --- a/tests/test_design_rules.py +++ b/tests/test_design_rules.py @@ -88,84 +88,12 @@ def frontend_name_branches(rel: str, tree: ast.Module) -> Iterator[Site]: _PORT_NAME = re.compile(r"(^|_)(port|PORT)(_BASE)?$") -def _subscriber_destination_arithmetic(tree: ast.Module) -> set[ast.BinOp]: - """Recognize outbound KV-event URLs through the Frontend subscriber contract. - - ``kv_events_subscriber`` returns a remote (host, port, topic), not a listener - allocation. A publisher may undo its engine's rank offset in that destination. - Follow single-assignment locals within one function, and exempt only the port - expression in an ``endpoint`` URL using that same remote host. Reusing the - port for a listener, or changing either local, must still trip the rule. - """ - destinations: set[ast.BinOp] = set() - for scope in ast.walk(tree): - if not isinstance(scope, ast.FunctionDef | ast.AsyncFunctionDef): - continue - # Nested scopes have independent bindings and are considered separately. - nodes: list[ast.AST] = [] - pending: list[ast.AST] = list(scope.body) - while pending: - node = pending.pop() - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef | ast.Lambda): - continue - nodes.append(node) - pending.extend(ast.iter_child_nodes(node)) - stores: dict[str, int] = {} - assignments: dict[str, ast.expr] = {} - for node in nodes: - if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store): - stores[node.id] = stores.get(node.id, 0) + 1 - if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): - assignments[node.targets[0].id] = node.value - addresses: set[tuple[str, str]] = set() - for node in nodes: - if not (isinstance(node, ast.Assign) and len(node.targets) == 1): - continue - target = node.targets[0] - if not (isinstance(target, ast.Tuple) and len(target.elts) == 3): - continue - host, port, _ = target.elts - if not (isinstance(host, ast.Name) and isinstance(port, ast.Name)): - continue - if stores.get(host.id) != 1 or stores.get(port.id) != 1: - continue - value = node.value - if isinstance(value, ast.Name) and stores.get(value.id) == 1: - value = assignments.get(value.id, value) - if isinstance(value, ast.IfExp) and isinstance(value.orelse, ast.Constant) and value.orelse.value is None: - value = value.body - if ( - isinstance(value, ast.Call) - and isinstance(value.func, ast.Attribute) - and value.func.attr == "kv_events_subscriber" - ): - addresses.add((host.id, port.id)) - for node in nodes: - if not isinstance(node, ast.Dict): - continue - for key, value in zip(node.keys, node.values, strict=True): - if not (isinstance(key, ast.Constant) and key.value == "endpoint" and isinstance(value, ast.JoinedStr)): - continue - match value.values: - case [ - ast.Constant(value="tcp://"), - ast.FormattedValue(value=ast.Name(id=host)), - ast.Constant(value=":"), - ast.FormattedValue(value=ast.BinOp(left=ast.Name(id=port)) as arithmetic), - ] if (host, port) in addresses: - destinations.add(arithmetic) - return destinations - - def port_arithmetic(rel: str, tree: ast.Module) -> Iterator[Site]: if rel in ("ports.py", "core/topology.py"): return - destinations = _subscriber_destination_arithmetic(tree) for node in ast.walk(tree): if not (isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add | ast.Sub)): continue - if node in destinations: - continue for operand in (node.left, node.right): dotted = _dotted(operand) if dotted is not None and _PORT_NAME.search(dotted.rsplit(".", 1)[-1]): From e238f6a3d4c2e384a8193ba97eafebfd4cccafec Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 12:58:05 -0500 Subject: [PATCH 24/25] docs(llm-d): pin upstream references to llm-d-router v0.10.0 The validated image ships the v0.10.0 EPP and P/D sidecar. Every flag, plugin type, metric and endpoint srtctl relies on exists at v0.10.0. Signed-off-by: Cam Quilici --- docs/llm-d.md | 28 ++++++++++++++-------------- src/srtctl/frontends/llm_d.py | 4 ++-- src/srtctl/services/llm_d_sidecar.py | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/llm-d.md b/docs/llm-d.md index c5d74813e..adb9b22bf 100644 --- a/docs/llm-d.md +++ b/docs/llm-d.md @@ -7,7 +7,7 @@ queue depth, load, ...) and, for prefill/decode, picks a prefill and a decode wo every request. No Dynamo, NATS, or etcd is involved. Upstream references are pinned to llm-d-router -[v0.11.0](https://github.com/llm-d/llm-d-router/tree/a5cbe600ebade00cf3e9885beaf2bfacddeabce1) +[v0.10.0](https://github.com/llm-d/llm-d-router/tree/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26) and the llm-d [no-Kubernetes guide](https://github.com/llm-d/llm-d/tree/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment). ## What runs where @@ -24,11 +24,11 @@ Envoy owns the public port. For every request it asks the EPP over ext_proc `x-gateway-destination-endpoint` header and Envoy's `ORIGINAL_DST` cluster forwards the request there. For prefill/decode the EPP picks a decode endpoint and names the chosen prefill worker in `x-prefiller-host-port` -([`disagg_profile_handler.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/epp/framework/plugins/scheduling/profilehandler/disagg/disagg_profile_handler.go#L533-L536)). +([`disagg_profile_handler.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/epp/framework/plugins/scheduling/profilehandler/disagg/disagg_profile_handler.go#L570-L573)). The decode endpoint is llm-d's P/D sidecar: it sends the request to that prefill worker first, then hands the returned `kv_transfer_params` to its own vLLM, whose KV connector pulls the cache from the prefill worker -([`connector_nixlv2.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/sidecar/proxy/connector_nixlv2.go#L42)). +([`connector_nixlv2.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/sidecar/proxy/connector_nixlv2.go#L54)). Everything else, `/metrics` included, the sidecar proxies to its vLLM. ### Why the EPP and Envoy are the frontend and the sidecar is a service @@ -43,7 +43,7 @@ the EPP and Envoy; no service phase sits between the workers and the frontend. The sidecar is a different process with its own image, placement, and health check, one per decode worker, on the worker's node; that is what `services` with `placement.per: worker` is for. Its `GET /health` answers 200 on its own -([`proxy.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/sidecar/proxy/proxy.go#L586-L588)), +([`proxy.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/sidecar/proxy/proxy.go#L624-L626)), so it starts in the `before_workers` phase and gates on that probe. It is critical: a dead sidecar takes its decode worker out of the routing pool. The frontend implies it, as Dynamo implies etcd; declare a service named `llm-d-sidecar` to change it: @@ -79,8 +79,8 @@ wide-EP guide) runs `nixlv2`. Any other decode connector is rejected at load tim ## Endpoint discovery The EPP runs without Kubernetes when its configuration names a discovery plugin -([`runner.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/cmd/epp/runner/runner.go#L264-L273), -[`runWithFileDiscovery`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/cmd/epp/runner/runner.go#L1044-L1196)). +([`runner.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/cmd/epp/runner/runner.go#L253-L261), +[`runWithFileDiscovery`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/cmd/epp/runner/runner.go#L1042-L1208)). srtctl writes three files into the log directory (every container sees it at `/logs`): - `llm-d-endpoints.yaml`: one endpoint per routable worker process (the one with an HTTP @@ -88,7 +88,7 @@ srtctl writes three files into the log directory (every container sees it at `/l interface, labelled `llm-d.ai/role: prefill`, `decode`, or `both` (aggregate). A decode endpoint is its sidecar's port. - `llm-d-epp-config.yaml`: `frontend.epp_config` with the - [`file-discovery`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/epp/framework/plugins/datalayer/discovery/file/plugin.go) + [`file-discovery`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/epp/framework/plugins/datalayer/discovery/file/plugin.go) plugin (`watchFile: false`) and `dataLayer.discovery` added. `discovery.pluginRef` is the spelling v0.10 and v0.11 both accept. A recipe that configures discovery itself is rejected. @@ -103,8 +103,8 @@ srtctl writes three files into the log directory (every container sees it at `/l The job is ready when Envoy's admin `/ready` answers 200 and the EPP's `llm_d_epp_ready_endpoints` gauge, the endpoints whose metrics it scraped within its staleness window -([`llm_d_router_metrics.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/epp/metrics/llm_d_router_metrics.go#L234-L241), -[`logger.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/epp/datalayer/logger/logger.go#L106-L117)), +([`llm_d_router_metrics.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/epp/metrics/llm_d_router_metrics.go#L238-L245), +[`logger.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/epp/datalayer/logger/logger.go#L105-L116)), counts every endpoint in the file. The EPP scrapes a worker on the address it routes to, so a decode worker is scraped through its sidecar. Tachometer scrapes the EPP's Prometheus listener as the frontend; worker metrics stay on vLLM's own port. @@ -122,8 +122,8 @@ for example on top of the vLLM image the workers run: ```dockerfile FROM vllm/vllm-openai: -COPY --from=ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.11.0 /app/epp /usr/local/bin/epp -COPY --from=ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.11.0 /app/pd-sidecar /usr/local/bin/pd-sidecar +COPY --from=ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.10.0 /app/epp /usr/local/bin/epp +COPY --from=ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.10.0 /app/pd-sidecar /usr/local/bin/pd-sidecar COPY --from=envoyproxy/envoy:distroless-v1.33.2 /usr/local/bin/envoy /usr/local/bin/envoy ``` @@ -188,7 +188,7 @@ roles: The approximate prefix scorers guess where a prefix is cached from the requests the EPP routed. With `roles..kv_events` the workers report it instead: every vLLM worker of the role publishes its KV-cache events (blocks stored and evicted) to the EPP's -[`precise-prefix-cache-producer`](https://github.com/llm-d/llm-d-router/tree/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/epp/framework/plugins/requestcontrol/dataproducer/preciseprefixcache), +[`precise-prefix-cache-producer`](https://github.com/llm-d/llm-d-router/tree/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/epp/framework/plugins/requestcontrol/dataproducer/preciseprefixcache), which indexes them per endpoint; `prefix-cache-scorer` with `prefixMatchInfoProducerName: precise-prefix-cache-producer` scores the endpoints from that index. The producer hashes the prompt's real token IDs, so the configuration also @@ -219,13 +219,13 @@ srtctl wires the rest: - The producer's `kvEventsConfig.zmqEndpoint` is `tcp://*:5557` (`discoverPods: false`): the EPP binds one ZMQ socket on the router node and every worker connects to it. The producer's per-pod mode dials the same port on every endpoint address, which workers - sharing a node cannot all bind (llm-d-router v0.11's file discovery gives every + sharing a node cannot all bind (llm-d-router v0.10's file discovery gives every endpoint rank 0). - Every publishing worker gets `--kv-events-config` with `endpoint: tcp://:5557` and `topic: kv@
:@`, naming the endpoint the EPP routes to (a decode worker's sidecar port) and the model requests name; the producer files the events under that endpoint - ([`vllm_adapter.go`](https://github.com/llm-d/llm-d-router/blob/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/kvevents/engineadapter/vllm_adapter.go#L52-L58)). + ([`vllm_adapter.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/kvevents/engineadapter/vllm_adapter.go#L52-L58)). vLLM [adds the publisher's DP rank to its port](https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/distributed/kv_events.py#L336-L340), so an external-LB rank `r` is given `5557 - r`. The engine sets [`data_parallel_index` to the global rank](https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/v1/engine/core.py#L1303-L1313), diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py index 09188f759..9f487a5eb 100644 --- a/src/srtctl/frontends/llm_d.py +++ b/src/srtctl/frontends/llm_d.py @@ -15,8 +15,8 @@ requires Envoy's ``/ready`` and EPP's ready-endpoint count, based on worker metrics. Optional ``roles..kv_events`` feed EPP's precise prefix-cache index. -Upstream, pinned: llm-d-router v0.11.0 -(https://github.com/llm-d/llm-d-router/tree/a5cbe600ebade00cf3e9885beaf2bfacddeabce1): +Upstream, pinned: llm-d-router v0.10.0 +(https://github.com/llm-d/llm-d-router/tree/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26): ``cmd/epp/runner/runner.go`` (``runWithFileDiscovery``), ``pkg/epp/framework/plugins/datalayer/discovery/file``; the Envoy configuration follows llm-d's no-Kubernetes guide diff --git a/src/srtctl/services/llm_d_sidecar.py b/src/srtctl/services/llm_d_sidecar.py index 0fef160a0..87b56be8d 100644 --- a/src/srtctl/services/llm_d_sidecar.py +++ b/src/srtctl/services/llm_d_sidecar.py @@ -10,8 +10,8 @@ It listens on ``Process.proxy_port`` and forwards other routes, including ``/metrics``, to the worker's HTTP port. -Upstream: llm-d-router ``cmd/pd-sidecar`` and ``pkg/sidecar/proxy`` at v0.11.0 -(https://github.com/llm-d/llm-d-router/tree/a5cbe600ebade00cf3e9885beaf2bfacddeabce1/pkg/sidecar/proxy). +Upstream: llm-d-router ``cmd/pd-sidecar`` and ``pkg/sidecar/proxy`` at v0.10.0 +(https://github.com/llm-d/llm-d-router/tree/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/sidecar/proxy). Its ``GET /health`` is independent of vLLM, allowing it to start before workers. """ From e4f5c0e85a4339b92fcdde799444b52a09deaaff Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 13:42:30 -0500 Subject: [PATCH 25/25] test: give worker-stage fake processes a dp_rank Signed-off-by: Cam Quilici --- tests/test_slurm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_slurm.py b/tests/test_slurm.py index 2ef1519df..2bf440437 100644 --- a/tests/test_slurm.py +++ b/tests/test_slurm.py @@ -231,6 +231,7 @@ def test_worker_stage_wraps_nonfatal_fingerprint_hook(tmp_path: Path) -> None: gpu_indices=list(range(8)), cuda_visible_devices="0,1,2,3,4,5,6,7", het_group=None, + dp_rank=None, trtllm_dist_init_port=29500, sidecar_grpc_port=50051, ) @@ -303,6 +304,7 @@ def _remap_worker_mixin(tmp_path: Path, *, frontend_type: str, dynamo_install: b gpu_indices=list(range(8)), cuda_visible_devices="0,1,2,3,4,5,6,7", het_group=None, + dp_rank=None, trtllm_dist_init_port=29500, sidecar_grpc_port=50051, ) @@ -883,6 +885,7 @@ def test_worker_stage_unsets_vllm_port_for_multinode_endpoint(tmp_path: Path) -> gpu_indices=list(range(8)), cuda_visible_devices="0,1,2,3,4,5,6,7", het_group=None, + dp_rank=None, trtllm_dist_init_port=29500, sidecar_grpc_port=50051, )