diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 034368670..a05963fe3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,5 +1,5 @@ # Default owners for all files -* @alec-flowers @nlevin-ui @csahithi +* @alec-flowers @ishandhanani @nlevin-ui @csahithi # Curated examples owners /examples/ @kyleliang-nv @hjjq @qiching @kedarpotdar-nv diff --git a/docs/cluster-config.md b/docs/cluster-config.md index 49fb752ed..ddb137d95 100644 --- a/docs/cluster-config.md +++ b/docs/cluster-config.md @@ -44,8 +44,6 @@ reporting: **output_dir**: When set, job logs are written to `output_dir/{job_id}/logs` instead of `srtctl_root/outputs/{job_id}/logs`. Useful for CI/CD and ephemeral environments. -**record_launch_plan**: Set `true` to record the realized launch plan (`output.record_launch_plan`) for every recipe on the cluster. Because the launch plan lives under the normal log directory, existing S3 postprocessing and external collectors that archive the complete log directory include it without a separate upload path. - **containers**: A map from alias to image path or registry URI. One resolver replaces image aliases in `model.container`, `roles..container`, frontend/benchmark images, exporter images and `services[].container`. Literal paths and registry URIs pass through untouched. Free-form maps (`environment`, `roles..env`, `roles..args`, `services[].env`, `container_mounts`) and the `identity` block are never rewritten. **default_bash_preamble**: A shell snippet (e.g. `"ulimit -n 1048576 -s unlimited -u 1048576"`) prepended to every container srun launched by srtctl: workers, frontends, telemetry, benchmark, postprocess. Runs before per-call `bash_preamble` and the main command, so cluster-wide ulimits apply to everything downstream. Silently dropped for distroless containers (e.g. `prom/node-exporter`) that bypass the bash wrapper; a WARNING log is emitted in that case. diff --git a/docs/config-reference.md b/docs/config-reference.md index 4b169b265..95c63bb5d 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -55,23 +55,15 @@ Fields: [ModelConfig](schema-reference.md#modelconfig). ## output -Output paths and optional reproducibility artifacts. +Output configuration with formattable paths. ```yaml output: log_dir: "./outputs/{job_id}/logs" - record_launch_plan: false # Save exact realized srun scripts and a manifest ``` The `log_dir` supports FormattablePath templating. See [FormattablePath Template System](runtime-env.md#formattablepath-template-system). -When `record_launch_plan` is enabled, srtctl creates `logs/launch-plan/manifest.json` and one executable shell -script for every realized `srun` invocation. Recording happens after Slurm assigns nodes, ports, heterogeneous -groups, mounts, and container paths, so the scripts describe what was actually launched rather than a pre-submit -estimate. Secret-like environment values are never persisted; the scripts name the environment variables that -must be supplied for replay. The resolved recipe, outer `sbatch_script.sh`, lockfile, and resource snapshot remain -alongside this directory and are referenced by the manifest. - ## health_check Health check configuration for worker readiness, and the worker log watch that fails a run early. diff --git a/docs/engines.md b/docs/engines.md index 68a333f5b..f29c742ae 100644 --- a/docs/engines.md +++ b/docs/engines.md @@ -144,7 +144,7 @@ Set `engine.dp_launch_mode: per_gpu` only when temporarily preserving the legacy When `TP x PP` fits on one node, srtslurm derives `--data-parallel-size-local` and `--data-parallel-start-rank`, then enables `--data-parallel-hybrid-lb` so every node-local process registers with the Dynamo frontend. When `TP x PP` is larger than the node-local GPU allocation, srtslurm instead derives the multi-node rendezvous arguments and makes every process except the global leader headless. For example, both DP4 x TP4 and DP2 x TP8 are selected automatically on four-GPU nodes. -Do not set `data-parallel-size-local`, `data-parallel-start-rank`, `data-parallel-hybrid-lb`, or `headless` manually; srtslurm owns those values. The allocation must be regular: `DP x TP x PP` must match the endpoint GPU count, and a TP/PP replica must divide evenly within or across nodes. +Do not set `data-parallel-size-local`, `data-parallel-start-rank`, `data-parallel-hybrid-lb`, or `headless` manually; srtslurm owns those values. Behind a router frontend, `data-parallel-external-lb: true` instead launches every DP rank as its own `vllm serve --data-parallel-rank ` on its own HTTP port, so the router addresses each rank ([llm-d](llm-d.md#routing-to-dp-ranks)). The allocation must be regular: `DP x TP x PP` must match the endpoint GPU count, and a TP/PP replica must divide evenly within or across nodes. ### TRT-LLM metrics publication diff --git a/docs/frontends.md b/docs/frontends.md index f962b2b4d..865607ebf 100644 --- a/docs/frontends.md +++ b/docs/frontends.md @@ -1,6 +1,6 @@ # Frontends and Dynamo -The `frontend:` block (which router fronts the workers) and the `dynamo:` block (how Dynamo is installed and wired). Router-specific pages: [SGLang Router](sglang-router.md), [vLLM Router](vllm-router.md), [Shepherd Model Gateway](smg.md). +The `frontend:` block (which router fronts the workers) and the `dynamo:` block (how Dynamo is installed and wired). Router-specific pages: [SGLang Router](sglang-router.md), [vLLM Router](vllm-router.md), [Shepherd Model Gateway](smg.md), [llm-d Router](llm-d.md). ## frontend @@ -8,7 +8,7 @@ Frontend/router configuration. ```yaml frontend: - # Frontend type: "dynamo" (default), "sglang-router", "vllm-router", "smg", direct "sglang", "vllm", "trtllm_serve", + # Frontend type: "dynamo" (default), "sglang-router", "vllm-router", "smg", "llm-d", direct "sglang", "vllm", "trtllm_serve", # or "none" for a services-only job (no router, no OpenAI endpoint, no worker-count health gate; only # valid without engine roles, see services[].nodes) type: dynamo diff --git a/docs/llm-d.md b/docs/llm-d.md new file mode 100644 index 000000000..adb9b22bf --- /dev/null +++ b/docs/llm-d.md @@ -0,0 +1,271 @@ +# llm-d router + +`frontend.type: llm-d` routes requests with the [llm-d](https://llm-d.ai) router: the +Endpoint Picker (EPP) behind Envoy, in front of direct `vllm serve` workers. The EPP +scores the workers with the plugins in `frontend.epp_config` (prefix-cache affinity, +queue depth, load, ...) and, for prefill/decode, picks a prefill and a decode worker for +every request. No Dynamo, NATS, or etcd is involved. + +Upstream references are pinned to llm-d-router +[v0.10.0](https://github.com/llm-d/llm-d-router/tree/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26) +and the llm-d [no-Kubernetes guide](https://github.com/llm-d/llm-d/tree/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment). + +## What runs where + +| Process | Where | Launched as | Port | +| --- | --- | --- | --- | +| Envoy | router node | the frontend (`llm-d-envoy_0`) | the public port (8000); admin `/ready` on 9901 | +| EPP | router node | the frontend (`llm-d-epp_0`) | ext_proc gRPC 9002, gRPC health 9003, Prometheus 9090 | +| P/D sidecar | every routable decode worker (prefill/decode jobs only) | the implied `llm-d-sidecar` service | `Process.proxy_port`, from the allocator | +| vLLM | every worker | the backend, as for the other static routers | `Process.http_port` | + +Envoy owns the public port. For every request it asks the EPP over ext_proc +(`FULL_DUPLEX_STREAMED`) which endpoint to use; the EPP sets the +`x-gateway-destination-endpoint` header and Envoy's `ORIGINAL_DST` cluster forwards the +request there. For prefill/decode the EPP picks a decode endpoint and names the chosen +prefill worker in `x-prefiller-host-port` +([`disagg_profile_handler.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/epp/framework/plugins/scheduling/profilehandler/disagg/disagg_profile_handler.go#L570-L573)). +The decode endpoint is llm-d's P/D sidecar: it sends the request to that prefill worker +first, then hands the returned `kv_transfer_params` to its own vLLM, whose KV connector +pulls the cache from the prefill worker +([`connector_nixlv2.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/sidecar/proxy/connector_nixlv2.go#L54)). +Everything else, `/metrics` included, the sidecar proxies to its vLLM. + +### Why the EPP and Envoy are the frontend and the sidecar is a service + +The frontend is the process that owns the public port and decides when the job is +ready, so Envoy and the EPP are the frontend. They also have to start at the frontend's +moment: llm-d discovers workers through Kubernetes or a file, and the workers here are +plain engines that register nowhere, so srtctl is the registrar. It writes the endpoints +file once every worker answers `/health` (the static routers' pre-start gate), then starts +the EPP and Envoy; no service phase sits between the workers and the frontend. + +The sidecar is a different process with its own image, placement, and health check, one +per decode worker, on the worker's node; that is what `services` with +`placement.per: worker` is for. Its `GET /health` answers 200 on its own +([`proxy.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/sidecar/proxy/proxy.go#L624-L626)), +so it starts in the `before_workers` phase and gates on that probe. It is critical: a +dead sidecar takes its decode worker out of the routing pool. The frontend implies it, +as Dynamo implies etcd; declare a service named `llm-d-sidecar` to change it: + +```yaml +services: + - name: llm-d-sidecar + type: llm-d-sidecar + container: my-llm-d-image # default: frontend.container_image, else the job container + args: + - --enable-prefiller-sampling # appended after the flags srtctl sets +``` + +srtctl sets `--port` (the worker's `Process.proxy_port`), `--model-server-port` (its +`Process.http_port`), `--kv-connector`, and `--secure-proxy=false`; overriding these +flags in `command` or `args` is rejected at config load. P/D jobs require exactly one +enabled sidecar service definition; customize the implied entry by name. +The allocator hands out the proxy port (`PROXY_PORTS`) for every routable worker of a +mode the frontend proxies (`Frontend.proxied_worker_modes`), so two decode workers on +one node never collide. The `--kv-connector` protocol comes from the decode workers' vLLM +connector class. Engine commands and sidecar selection use the same payload: +a role's explicit `kv-transfer-config` takes precedence over its `connector`, +which takes precedence over `engine.connector`: + +| Decode connector | Sidecar protocol | +| --- | --- | +| `NixlConnector` | `nixlv2` | + +A `MultiConnector` maps through the first connector it wraps that has a protocol, so +NIXL next to vLLM's `OffloadingConnector` (CPU KV offloading, as in llm-d's tiered +wide-EP guide) runs `nixlv2`. Any other decode connector is rejected at load time. + +## Endpoint discovery + +The EPP runs without Kubernetes when its configuration names a discovery plugin +([`runner.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/cmd/epp/runner/runner.go#L253-L261), +[`runWithFileDiscovery`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/cmd/epp/runner/runner.go#L1042-L1208)). +srtctl writes three files into the log directory (every container sees it at `/logs`): + +- `llm-d-endpoints.yaml`: one endpoint per routable worker process (the one with an HTTP + port: a multi-node worker's leader), at its IPv4 address on the cluster's network + interface, labelled `llm-d.ai/role: prefill`, `decode`, or `both` (aggregate). A decode + endpoint is its sidecar's port. +- `llm-d-epp-config.yaml`: `frontend.epp_config` with the + [`file-discovery`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/epp/framework/plugins/datalayer/discovery/file/plugin.go) + plugin (`watchFile: false`) and `dataLayer.discovery` added. `discovery.pluginRef` is + the spelling v0.10 and v0.11 both accept. A recipe that configures discovery itself is + rejected. +- `llm-d-envoy.yaml`: the guide's Envoy configuration with the public port, an admin + listener srtctl can reach, no request timeout, `/metrics` answered with 404 (routed, it + would return a different worker's counters on every scrape), and an access log + (`llm-d-envoy-access.log`) recording, for every request, the endpoint the EPP picked + (`upstream=`) and the prefill worker it named (`prefiller=`). + +## Readiness and metrics + +The job is ready when Envoy's admin `/ready` answers 200 and the EPP's +`llm_d_epp_ready_endpoints` gauge, the endpoints whose metrics it scraped within its +staleness window +([`llm_d_router_metrics.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/epp/metrics/llm_d_router_metrics.go#L238-L245), +[`logger.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/epp/datalayer/logger/logger.go#L105-L116)), +counts every endpoint in the file. The EPP scrapes a worker on the address it routes +to, so a decode worker is scraped through its sidecar. Tachometer scrapes the EPP's +Prometheus listener as the frontend; worker metrics stay on vLLM's own port. + +## Images + +The EPP, Envoy, and the sidecar run from `frontend.container_image` (the sidecar unless +its service sets `container`), or the job container when it is unset, as the executables +`epp`, `envoy`, and `pd-sidecar` on `PATH`. llm-d publishes the EPP and the sidecar as +distroless images (`/app/epp`, `/app/pd-sidecar`), which pyxis cannot start: it enters a +container through `sh` +([`pyxis_slurmstepd.c`](https://github.com/NVIDIA/pyxis/blob/107519944221822ea1dace4db8e7234b2eaa4cd5/pyxis_slurmstepd.c#L1128-L1135)). Build an image with a shell that carries the three executables, +for example on top of the vLLM image the workers run: + + +```dockerfile +FROM vllm/vllm-openai: +COPY --from=ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.10.0 /app/epp /usr/local/bin/epp +COPY --from=ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.10.0 /app/pd-sidecar /usr/local/bin/pd-sidecar +COPY --from=envoyproxy/envoy:distroless-v1.33.2 /usr/local/bin/envoy /usr/local/bin/envoy +``` + +## Recipe + +```yaml +frontend: + type: llm-d + enable_multiple_frontends: false + container_image: llm-d-vllm # optional: the router image + epp_config: # EndpointPickerConfig without discovery + plugins: [...] + schedulingProfiles: [...] + args: # extra EPP flags, e.g. log verbosity + v: 2 +engine: + type: vllm + connector: nixl +``` + +`frontend.epp_config` is accepted only with `frontend.type: llm-d`. +`frontend.args` are passed to the EPP; overriding the pool, configuration, port, or +`secure-serving` flags is rejected at config load. A prefill/decode `epp_config` needs +`prefill` and `decode` scheduling profiles and the `disagg-profile-handler`. +Without `epp_config`, srtctl runs the scorers of llm-d's +no-Kubernetes guide +([`config.yaml`](https://github.com/llm-d/llm-d/blob/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment/router/epp/config.yaml)): +queue depth (2), KV-cache utilization (2), approximate prefix cache (3), and +no-hit LRU (2). A prefill/decode job gets them as a `prefill` profile, a `decode` +profile that scores by queue depth and KV-cache utilization alone, and the +`disagg-profile-handler` with `always-disagg-pd-decider`. (The EPP's own default applies +only to a configuration file without plugins, and srtctl's always carries file +discovery.) See +[`examples/vllm/llm-d-disagg.yaml`](https://github.com/NVIDIA/srt-slurm/blob/main/examples/vllm/llm-d-disagg.yaml) and +[`examples/vllm/llm-d-agg.yaml`](https://github.com/NVIDIA/srt-slurm/blob/main/examples/vllm/llm-d-agg.yaml). + +## Routing to DP ranks + +A vLLM worker with `data-parallel-size` is one `vllm serve` that balances its DP ranks +itself (the leader serves the API, other nodes run headless ranks), so it is one EPP +endpoint, as it is one SMG worker. Set `data-parallel-external-lb: true` in the role's +`args` to have the EPP route to the ranks themselves, as llm-d's wide-EP guide does: srtctl +then launches every DP rank as its own `vllm serve --data-parallel-rank +--data-parallel-address ` (vLLM's external load balancing) on its own +allocated HTTP port, on one node or across several, and lists each rank in the endpoints +file. Each rank's step and log carry `_dp` (`agg_0__dp3`, `_agg_w0_dp3.out`), +and in a prefill/decode job every decode rank gets its own P/D sidecar. vLLM accepts +external load balancing for MoE models only. See +[`examples/vllm/llm-d-dp-ranks.yaml`](https://github.com/NVIDIA/srt-slurm/blob/main/examples/vllm/llm-d-dp-ranks.yaml). + +```yaml +roles: + agg: + gpus: 16 + args: + data-parallel-size: 16 + data-parallel-external-lb: true +``` + +## Precise prefix-cache routing + +The approximate prefix scorers guess where a prefix is cached from the requests the EPP +routed. With `roles..kv_events` the workers report it instead: every vLLM worker of +the role publishes its KV-cache events (blocks stored and evicted) to the EPP's +[`precise-prefix-cache-producer`](https://github.com/llm-d/llm-d-router/tree/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/epp/framework/plugins/requestcontrol/dataproducer/preciseprefixcache), +which indexes them per endpoint; `prefix-cache-scorer` with +`prefixMatchInfoProducerName: precise-prefix-cache-producer` scores the endpoints from +that index. The producer hashes the prompt's real token IDs, so the configuration also +needs a `token-producer`, which tokenizes through a worker's `/v1/*/render` endpoints: + +```yaml +frontend: + epp_config: + plugins: + - type: token-producer + - type: precise-prefix-cache-producer + parameters: + indexerConfig: + kvBlockIndexConfig: + enableMetrics: true # index admissions and lookup hits on /metrics + - type: prefix-cache-scorer + parameters: + prefixMatchInfoProducerName: precise-prefix-cache-producer + - type: max-score-picker + schedulingProfiles: [...] +roles: + agg: + kv_events: true +``` + +srtctl wires the rest: + +- The producer's `kvEventsConfig.zmqEndpoint` is `tcp://*:5557` (`discoverPods: false`): + the EPP binds one ZMQ socket on the router node and every worker connects to it. The + producer's per-pod mode dials the same port on every endpoint address, which workers + sharing a node cannot all bind (llm-d-router v0.10's file discovery gives every + endpoint rank 0). +- Every publishing worker gets `--kv-events-config` with `endpoint: + tcp://:5557` and `topic: kv@
:@`, naming the + endpoint the EPP routes to (a decode worker's sidecar port) and the model requests + name; the producer files the events under that endpoint + ([`vllm_adapter.go`](https://github.com/llm-d/llm-d-router/blob/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/kvevents/engineadapter/vllm_adapter.go#L52-L58)). + vLLM [adds the publisher's DP rank to its port](https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/distributed/kv_events.py#L336-L340), + so an external-LB rank `r` is given `5557 - r`. The engine sets + [`data_parallel_index` to the global rank](https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/v1/engine/core.py#L1303-L1313), + which the [scheduler passes to the publisher](https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/v1/core/sched/scheduler.py#L161-L164). + An explicit TCP address [connects rather than binds](https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/distributed/kv_events.py#L410-L427), + so all ranks reach the EPP's one subscriber socket. +- The token producer's `vllm.url` is the first prefill worker's (else the first worker's) + own vLLM API, and `modelName` defaults to the served model name. + +These vLLM references are pinned to `ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9`, +shipped in the tested `quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly` image. +The [run artifact](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/37396765122/artifacts/11388917384) +records `VLLM_IMAGE_TAG=vllm/vllm-openai:nightly-ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9` +and installed version `0.27.2rc1.dev77+gac7509e2b` in `fingerprint_decode_w0.json`. + +With vLLM's `OffloadingConnector` and its `self_describing_kv_events: true`, a worker also +reports the blocks it offloads to host memory; the producer indexes them as the `cpu` tier +and weights tiers by `indexerConfig.kvCacheBackendConfigs` (`gpu` 1.0, `cpu` 0.8 by +default). + +The roles that publish and the producer go together: either without the other is +rejected, as are a producer `kvEventsConfig` that sets the socket or discovery, a +`token-producer` `vllm.url`, worker `kv_events.endpoint` or `kv_events.topic`, and a +router not on the head node. The tokenizer URL is managed even without precise routing. +A role with `data-parallel-size` publishes only with `data-parallel-external-lb: true`: each DP rank +publishes its own events, and the EPP can place a prefix only on an endpoint that is +that rank. + +## Limitations + +- vLLM workers over HTTP only; gRPC roles are rejected. +- One router replica: the EPP keeps its scheduling state (the approximate prefix index, + in-flight load) in memory. Multi-node jobs require `enable_multiple_frontends: false`: + nginx would put the router on a different node from the public endpoint, where readiness + checks expect Envoy's admin and the EPP's metrics listeners. Setting + `num_additional_frontends: 0` still enables nginx and is rejected too. Single-node jobs + never launch nginx, regardless of `enable_multiple_frontends`. +- The endpoints file is written once (`watchFile: false`) and the EPP does not eject a + worker that fails; a crashed worker or sidecar fails the job instead. +- vLLM's `data-parallel-multi-port-external-lb` (one supervisor per node serving a port + per rank) is not used: its ranks publish KV events on per-rank ports that file + discovery can only reach with llm-d-router's per-endpoint `rankIndex` (after v0.11.0). +- The precise prefix index lives in the one EPP; KV events go to that EPP alone. diff --git a/docs/schema-reference.md b/docs/schema-reference.md index 717531540..0e13be02a 100644 --- a/docs/schema-reference.md +++ b/docs/schema-reference.md @@ -121,7 +121,7 @@ Frontend/router configuration. | Key | Type | Default | Description | |---|---|---|---| -| `type` | str | `'dynamo'` | Frontend type - "dynamo" (default); "sglang-router" (SGLang Model Gateway), "vllm-router", "smg" (Shepherd Model Gateway, any backend), "atomesh", and "tilert-router" (static routers); "sglang", "vllm", and "trtllm_serve" (direct: the single aggregate worker binds the public port, no router process); "none" (services-only job: no router, no OpenAI endpoint, no worker-count health gate; requires no engine roles). Pre-2.0 recipes spelled the router "sglang"; ``srtctl migrate`` rewrites that to "sglang-router". | +| `type` | str | `'dynamo'` | Frontend type - "dynamo" (default); "sglang-router" (SGLang Model Gateway), "vllm-router", "smg" (Shepherd Model Gateway, any backend), "llm-d" (llm-d Endpoint Picker behind Envoy, vLLM), "atomesh", and "tilert-router" (static routers); "sglang", "vllm", and "trtllm_serve" (direct: the single aggregate worker binds the public port, no router process); "none" (services-only job: no router, no OpenAI endpoint, no worker-count health gate; requires no engine roles). Pre-2.0 recipes spelled the router "sglang"; ``srtctl migrate`` rewrites that to "sglang-router". | | `enable_multiple_frontends` | bool | `True` | Scale with nginx + multiple routers. When ``True`` (default), srtctl stands up nginx and fans out to ``num_additional_frontends + 1`` router replicas. When ``False``, there is NO nginx proxy — the benchmark must target the single master router (or a worker) directly at ``http://localhost:``. ``benchmark.command`` has no placeholder substitution, so write the URL out literally. | | `num_additional_frontends` | int | `9` | Additional routers beyond master (default: 9) | | `nginx_container` | str | `'nginx:1.27.4'` | Custom nginx container image (default: nginx:1.27.4) | @@ -130,6 +130,7 @@ Frontend/router configuration. | `nginx_session_affinity_header` | str | `'X-Dynamo-Session-ID'` | Header hashed when affinity is on (default ``X-Dynamo-Session-ID``). Set ``X-Correlation-ID`` for clients (e.g. aiperf) that carry the session id in that header instead. | | `nginx_keepalive_timeout` | str | `'600s'` | Idle timeout for client and upstream keepalive connections in the generated nginx.conf (default "600s"). nginx's own default is 75s, which closes a session's connection during the long recorded think-time of an agentic replay; the client's next write on that pooled socket then fails with "broken pipe" / "server disconnected" and nothing is logged server-side. | | `worker_selection` | dict[str, Any] \| None | `None` | Inline Dynamo worker-selection policy configuration. srtctl writes this mapping under the top-level ``worker_selection`` key in a generated router policy YAML and passes it to the Dynamo frontend via ``--router-policy-config``. | +| `epp_config` | dict[str, Any] \| None | `None` | llm-d Endpoint Picker configuration (``EndpointPickerConfig``: ``plugins``, ``schedulingProfiles``, ...) for ``frontend.type: llm-d``. srtctl writes it to a file with the ``file-discovery`` plugin and ``dataLayer.discovery`` added, so the EPP reads the job's workers from the endpoints file srtctl renders. Omitted, srtctl runs the scorers of llm-d's no-Kubernetes guide (in prefill and decode profiles for a prefill/decode job). | | `args` | dict[str, Any] \| None | `None` | CLI arguments passed to the frontend/router process | | `env` | dict[str, str] \| None | `None` | Environment variables for frontend processes | | `container_image` | str \| None | `None` | Optional router-specific image. Static routers use the model/backend image when omitted. | @@ -248,12 +249,11 @@ Profiling configuration. ### OutputConfig -Output paths and optional reproducibility artifacts. +Output configuration with formattable paths. | Key | Type | Default | Description | |---|---|---|---| | `log_dir` | [FormattablePath](#formattablepath) | `()` | Directory for job logs and results; a FormattablePath, so `{job_id}` and `$VARS` expand. | -| `record_launch_plan` | bool | `False` | Save the realized `srun` scripts and a manifest under `logs/launch-plan/`. | ### HealthCheckConfig @@ -800,7 +800,6 @@ Top-level keys of `srtslurm.yaml`. Recipes inherit these defaults and resolve al | `default_health_check` | dict[str, int] \| None | `None` | `health_check` block (`max_attempts`, `interval_seconds`) used when a recipe has none. | | `srtctl_root` | str \| None | `None` | srtctl checkout on the shared filesystem that compute nodes mount at /srtctl-src. Default: this checkout. | | `output_dir` | str \| None | `None` | Custom output directory for job logs | -| `record_launch_plan` | bool | `False` | Cluster-wide default for recording exact realized srun commands. Recipes can opt in independently with output.record_launch_plan. | | `model_paths` | dict[str, str] \| None | `None` | Alias -> path map; a recipe's `model.path` may name an alias instead of a path. | | `containers` | dict[str, str] \| None | `None` | Alias -> image map, resolved for every container key in a recipe (`model.container`, `roles..container`, ...). | | `cloud` | dict[str, str] \| None | `None` | Free-form cloud settings. Accepted for compatibility; srtctl does not read it. | diff --git a/docs/schema/cluster.schema.json b/docs/schema/cluster.schema.json index cffd4cc13..3b3f01d95 100644 --- a/docs/schema/cluster.schema.json +++ b/docs/schema/cluster.schema.json @@ -178,11 +178,6 @@ "description": "Custom output directory for job logs", "default": null }, - "record_launch_plan": { - "type": "boolean", - "description": "Cluster-wide default for recording exact realized srun commands. Recipes can opt in independently with output.record_launch_plan.", - "default": false - }, "model_paths": { "anyOf": [ { diff --git a/docs/schema/recipe.schema.json b/docs/schema/recipe.schema.json index 3a46a60d8..0f0044165 100644 --- a/docs/schema/recipe.schema.json +++ b/docs/schema/recipe.schema.json @@ -868,7 +868,7 @@ "properties": { "type": { "type": "string", - "description": "Frontend type - \"dynamo\" (default); \"sglang-router\" (SGLang Model Gateway), \"vllm-router\", \"smg\" (Shepherd Model Gateway, any backend), \"atomesh\", and \"tilert-router\" (static routers); \"sglang\", \"vllm\", and \"trtllm_serve\" (direct: the single aggregate worker binds the public port, no router process); \"none\" (services-only job: no router, no OpenAI endpoint, no worker-count health gate; requires no engine roles). Pre-2.0 recipes spelled the router \"sglang\"; ``srtctl migrate`` rewrites that to \"sglang-router\".", + "description": "Frontend type - \"dynamo\" (default); \"sglang-router\" (SGLang Model Gateway), \"vllm-router\", \"smg\" (Shepherd Model Gateway, any backend), \"llm-d\" (llm-d Endpoint Picker behind Envoy, vLLM), \"atomesh\", and \"tilert-router\" (static routers); \"sglang\", \"vllm\", and \"trtllm_serve\" (direct: the single aggregate worker binds the public port, no router process); \"none\" (services-only job: no router, no OpenAI endpoint, no worker-count health gate; requires no engine roles). Pre-2.0 recipes spelled the router \"sglang\"; ``srtctl migrate`` rewrites that to \"sglang-router\".", "default": "dynamo" }, "enable_multiple_frontends": { @@ -918,6 +918,18 @@ "description": "Inline Dynamo worker-selection policy configuration. srtctl writes this mapping under the top-level ``worker_selection`` key in a generated router policy YAML and passes it to the Dynamo frontend via ``--router-policy-config``.", "default": null }, + "epp_config": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "description": "llm-d Endpoint Picker configuration (``EndpointPickerConfig``: ``plugins``, ``schedulingProfiles``, ...) for ``frontend.type: llm-d``. srtctl writes it to a file with the ``file-discovery`` plugin and ``dataLayer.discovery`` added, so the EPP reads the job's workers from the endpoints file srtctl renders. Omitted, srtctl runs the scorers of llm-d's no-Kubernetes guide (in prefill and decode profiles for a prefill/decode job).", + "default": null + }, "args": { "anyOf": [ { @@ -1547,14 +1559,9 @@ "log_dir": { "type": "string", "description": "Directory for job logs and results; a FormattablePath, so `{job_id}` and `$VARS` expand." - }, - "record_launch_plan": { - "type": "boolean", - "description": "Save the realized `srun` scripts and a manifest under `logs/launch-plan/`.", - "default": false } }, - "description": "Output paths and optional reproducibility artifacts." + "description": "Output configuration with formattable paths." }, "PlacementConfig": { "type": "object", diff --git a/docs/services.md b/docs/services.md index 4b4b76bb5..0105ba292 100644 --- a/docs/services.md +++ b/docs/services.md @@ -48,7 +48,7 @@ directory. `examples/features/services.yaml` is a runnable version of this. ```yaml services: - name: my-sidecar # required, unique across the list - type: generic # generic (default) | etcd | nats | mooncake-master | dcgm-exporter | node-exporter | mooncake-store | lmcache-server + type: generic # generic (default) | etcd | nats | mooncake-master | dcgm-exporter | node-exporter | mooncake-store | lmcache-server | llm-d-sidecar enabled: true # false drops the service (the way to switch an implied one off) external: null # typed kinds only: use an instance that already runs at this address command: # argv, not shell-interpreted; required for generic @@ -135,6 +135,7 @@ Things the recipe asks for elsewhere are services the job runs without an entry; | a declared `mooncake-master` entry (see [Mooncake KV Store](mooncake-kv-store.md)) | `mooncake-master` | the infra node, phase `before_workers` | | a vLLM role with `connector: lmcache-mp` (engine-level or `roles..args.connector`), unless a `lmcache-server` entry is declared | `lmcache-server` | that role's nodes, or every worker node when several roles use it, phase `before_workers` | | `engine.failover` (see [Shadow Engine Recovery](shadow-engine-recovery.md)) | `gms` | one instance per vLLM worker (`placement.per: worker`), phase `before_workers` | +| `frontend.type: llm-d` with prefill and decode workers (see [llm-d Router](llm-d.md)) | `llm-d-sidecar` | one instance per routable decode worker (`placement.per: worker`), phase `before_workers` | | tachometer on (the default; `observability.tachometer.enabled`) | `dcgm-exporter`, `node-exporter` | every worker node, phase `after_frontend` | `srtctl dry-run` lists them next to the declared ones, marked `implied by:`. A declared entry with @@ -191,7 +192,7 @@ node are rejected before anything launches; give them disjoint placements or por of the placed role(s) on each selected node (`node` must be `prefill`, `decode`, `agg`, or `workers`), a sidecar in the Kubernetes sense. The instance runs in that worker's device view (its `CUDA_VISIBLE_DEVICES`, the same pinning the worker's engine steps get) and sees `{worker_role}`, -`{worker_index}`, `{worker_node_rank}`, `{worker_gpus}` and `{worker_gpu_count}`. Its step and log are +`{worker_index}`, `{worker_node_rank}`, `{worker_gpus}` and `{worker_gpu_count}`, plus `{worker_http_port}` when the worker serves an API and `{worker_proxy_port}` when the frontend reaches it through a proxy (`Process.proxy_port`). The kind decides which workers get an instance: every one by default; the `llm-d-sidecar` only the routable decode workers it fronts. Its step and log are `service____`. Because the instances of one node share its network namespace, a per-worker service's `readiness` must be a log probe. The GPU Memory Service of [shadow engine recovery](shadow-engine-recovery.md) is the built-in per-worker kind, and [SGLang fast engine recovery](sglang-weight-cache.md) runs SGLang's weight cache daemon as a plain `generic` per-worker service: @@ -305,6 +306,7 @@ environment its process needs; the launch path is shared by every kind. Register | `mooncake-store` | `python -m mooncake.mooncake_store_service` | `before_workers` | `true` | Requires a `mooncake-master` entry. Container falls back to the master's. Injects the master's address. | | `lmcache-server` | `lmcache server` bound to the RPC and HTTP ports srtctl owns (8750, 8751) | `before_workers` | `true` | One per worker node (`placement.node: workers`); vLLM ranks reach it over localhost with `connector: lmcache-mp`, which implies it. SGLang workers started with `enable-lmcache` get `LMCACHE_MP_HOST`/`LMCACHE_MP_PORT` pointing at it unless the recipe sets `lmcache-config-file` or `LMCACHE_MP_HOST`. Services do not inherit the top-level `environment:`, so set `PYTHONHASHSEED` in its `env:` when the workers set it. `args` are appended (`--l1-size-gb`, `--chunk-size`, `--max-workers`, ...). Ready when `GET /healthcheck` answers; LMCache must be installed in the job container. | | `ray` | `ray start --head ...` on the first instance, `ray start --address=:6379 ...` on the rest, both `--block` | `before_workers` | `true` | One Ray cluster across the service's nodes; see [Ray cluster](#ray-cluster). Placement `workers` (default), `head`, or `all`. `options`: `port` (GCS, 6379), `dashboard_port` (8265), `num_gpus` (the node's count). `args` are appended to every `ray start`. | +| `llm-d-sidecar` | `pd-sidecar --port= --model-server-port= --kv-connector= --secure-proxy=false` in `frontend.container_image`, else the job container | `before_workers` | `true` | Implied by `frontend.type: llm-d` for prefill/decode; see [llm-d Router](llm-d.md). `placement.per: worker` on decode: one instance per routable decode worker, bound to its allocated `Process.proxy_port`. `command` replaces the executable, `args` are appended. Ready when `GET /health` on its port answers. | | `gms` | one `python3 -m gpu_memory_service --device k` per GPU of the worker, supervised by bash, in the job container | `before_workers` | `true` | Implied by `engine.failover`; see [Shadow Engine Recovery](shadow-engine-recovery.md). `placement.per: worker` (required): one instance per vLLM worker in that worker's device view, sockets and lock file under `/srtctl-/_`. Ready when its log says `GMS ready:`; `readiness.timeout_seconds` (default 120) also bounds the servers' startup. | ### Ray cluster diff --git a/examples/README.md b/examples/README.md index 8b7bcb0b1..c936e4d26 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,7 +7,7 @@ Small starting points, one per frontend and topology. The matrix examples serve | Backend | Dynamo frontend | Native router | Router-free direct | | --- | --- | --- | --- | | SGLang | `sglang/dynamo-agg.yaml`, `sglang/dynamo-disagg.yaml` | `sglang/sglang-router-agg.yaml`, `sglang/sglang-router-disagg.yaml`, `sglang/smg-disagg.yaml` (SMG) | `sglang/sglang-direct-agg.yaml` | -| vLLM | `vllm/dynamo-agg.yaml`, `vllm/dynamo-disagg.yaml` | `vllm/vllm-router-agg.yaml`, `vllm/vllm-router-disagg.yaml`, `vllm/vllm-router-moriio-disagg.yaml` (ROCm, MoRI-IO discovery), `vllm/smg-agg.yaml`, `vllm/smg-dep16.yaml`, `vllm/smg-disagg-grpc.yaml` (SMG; P/D over gRPC with NIXL) | `vllm/vllm-direct-agg.yaml` | +| vLLM | `vllm/dynamo-agg.yaml`, `vllm/dynamo-disagg.yaml` | `vllm/vllm-router-agg.yaml`, `vllm/vllm-router-disagg.yaml`, `vllm/vllm-router-moriio-disagg.yaml` (ROCm, MoRI-IO discovery), `vllm/smg-agg.yaml`, `vllm/smg-dep16.yaml`, `vllm/smg-disagg-grpc.yaml` (SMG; P/D over gRPC with NIXL), `vllm/llm-d-agg.yaml`, `vllm/llm-d-disagg.yaml` (llm-d EPP behind Envoy; P/D through the llm-d sidecar), `vllm/llm-d-dp-ranks.yaml` (llm-d routing to every external-LB DP rank by its KV-cache events) | `vllm/vllm-direct-agg.yaml` | | TRT-LLM | `trtllm/dynamo-agg.yaml`, `trtllm/dynamo-disagg.yaml`, `trtllm/dynamo-disagg-multinode.yaml` (TP16 decode across two 8-GPU nodes: one srun, one host per rank under `--distribution=arbitrary`) | `trtllm/trtllm-serve-disagg.yaml`, `trtllm/smg-agg.yaml` (SMG) | `trtllm/trtllm-serve-agg.yaml` | | TokenSpeed | `tokenspeed/dynamo-agg.yaml`, `tokenspeed/dynamo-disagg.yaml` | `tokenspeed/smg-agg.yaml`, `tokenspeed/smg-disagg.yaml` (SMG, gRPC engines) | | | Mocker | `mocker/dynamo-agg.yaml` | | | @@ -63,6 +63,7 @@ containers: vllm-lmcache: /path/to/vllm-lmcache.sqsh # vLLM image with LMCache installed, for features/lmcache-server.yaml and -disagg.yaml sglang-lmcache: /path/to/sglang-lmcache.sqsh # SGLang image with LMCache installed, for features/lmcache-server-sglang.yaml smg: /path/to/smg.sqsh # Shepherd Model Gateway image (lightseekorg/smg), the router for the smg examples + llm-d-vllm: /path/to/llm-d-vllm.sqsh # vLLM image with epp, pd-sidecar and envoy on PATH, for the llm-d examples (docs/llm-d.md#images) ``` ## Optional GPU temperature diff --git a/examples/vllm/llm-d-agg.yaml b/examples/vllm/llm-d-agg.yaml new file mode 100644 index 000000000..fc69f0589 --- /dev/null +++ b/examples/vllm/llm-d-agg.yaml @@ -0,0 +1,63 @@ +# Two aggregated vLLM workers behind the llm-d router (docs/llm-d.md). +# +# On the router node srtctl starts llm-d's Endpoint Picker (EPP) and Envoy on the +# public port; the EPP reads both workers from an endpoints file srtctl writes and +# picks one per request with the scorers below. +# +# Aliases (`qwen3-0.6b`, `llm-d-vllm`) resolve through srtslurm.yaml; see +# examples/README.md. `llm-d-vllm` is a vLLM image that also ships the `epp`, +# `pd-sidecar`, and `envoy` executables on PATH (docs/llm-d.md#images). + +schema: 2 +name: "qwen3-0.6b-vllm-llm-d-agg" + +slurm: + time_limit: "00:30:00" + +model: + path: "qwen3-0.6b" + container: "llm-d-vllm" + precision: "bf16" + +resources: + gpu_type: "h100" + gpus_per_node: 8 +frontend: + type: llm-d + enable_multiple_frontends: false + # Prefix-cache affinity weighted over queue depth; srtctl adds the file-discovery plugin. + epp_config: + plugins: + # Hashes each prompt's blocks into the prefix index the prefix-cache scorer reads. + - type: approx-prefix-cache-producer + - type: prefix-cache-scorer + - type: queue-scorer + - type: max-score-picker + schedulingProfiles: + - name: default + plugins: + - pluginRef: prefix-cache-scorer + weight: 3 + - pluginRef: queue-scorer + weight: 2 + - pluginRef: max-score-picker + +engine: vllm +roles: + agg: + nodes: 1 + workers: 2 + gpus: 1 + env: + PYTHONUNBUFFERED: "1" + args: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + gpu-memory-utilization: 0.5 + max-model-len: 4096 + +benchmark: + type: "sa-bench" + isl: 128 + osl: 128 + concurrencies: "4x8" diff --git a/examples/vllm/llm-d-disagg.yaml b/examples/vllm/llm-d-disagg.yaml new file mode 100644 index 000000000..d3df5ba59 --- /dev/null +++ b/examples/vllm/llm-d-disagg.yaml @@ -0,0 +1,94 @@ +# vLLM disaggregated prefill/decode behind the llm-d router (docs/llm-d.md). +# +# One TP1 prefill and one TP1 decode `vllm serve` worker sharing a node, KV over +# NIXL. On the router node srtctl starts llm-d's Endpoint Picker (EPP), which +# reads the workers from an endpoints file srtctl writes, and Envoy on the public +# port. The decode worker sits behind llm-d's P/D sidecar, a service the +# frontend implies (`llm-d-sidecar`, one per decode worker, started before the +# workers): the EPP picks the decode sidecar and a prefill worker for each +# request, and the sidecar runs the prefill before handing the KV transfer +# parameters to its vLLM. +# +# Aliases (`qwen3-0.6b`, `llm-d-vllm`) resolve through srtslurm.yaml; see +# examples/README.md. `llm-d-vllm` is a vLLM image that also ships the `epp`, +# `pd-sidecar`, and `envoy` executables on PATH (docs/llm-d.md#images). + +schema: 2 +name: "qwen3-0.6b-vllm-llm-d-disagg" + +slurm: + time_limit: "00:30:00" + +model: + path: "qwen3-0.6b" + container: "llm-d-vllm" + precision: "bf16" + +resources: + gpu_type: "h100" + gpus_per_node: 8 +frontend: + type: llm-d + enable_multiple_frontends: false + # The EPP's scheduler: decode and prefill profiles, picked by the disaggregation + # profile handler; srtctl adds the file-discovery plugin. + epp_config: + plugins: + # Hashes each prompt's blocks into the prefix index the prefix-cache scorer reads. + - type: approx-prefix-cache-producer + - type: prefill-filter + - type: decode-filter + - type: prefix-cache-scorer + - type: queue-scorer + - type: always-disagg-pd-decider + - type: disagg-profile-handler + parameters: + deciders: + prefill: always-disagg-pd-decider + - type: max-score-picker + schedulingProfiles: + - name: prefill + plugins: + - pluginRef: prefill-filter + - pluginRef: prefix-cache-scorer + weight: 2 + - pluginRef: queue-scorer + - pluginRef: max-score-picker + - name: decode + plugins: + - pluginRef: decode-filter + - pluginRef: queue-scorer + - pluginRef: max-score-picker + +engine: + type: vllm + connector: nixl +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 1 + env: + PYTHONUNBUFFERED: "1" + args: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + gpu-memory-utilization: 0.5 + max-model-len: 4096 + decode: + nodes: colocate + workers: 1 + gpus: 1 + env: + PYTHONUNBUFFERED: "1" + args: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + gpu-memory-utilization: 0.5 + max-model-len: 4096 + +benchmark: + type: "sa-bench" + isl: 128 + osl: 128 + concurrencies: "4x8" diff --git a/examples/vllm/llm-d-dp-ranks.yaml b/examples/vllm/llm-d-dp-ranks.yaml new file mode 100644 index 000000000..4ef980b27 --- /dev/null +++ b/examples/vllm/llm-d-dp-ranks.yaml @@ -0,0 +1,75 @@ +# vLLM data parallel 16 with expert parallelism across two nodes, behind the llm-d +# router, which routes to every DP rank by the KV-cache blocks it holds (docs/llm-d.md). +# +# `data-parallel-external-lb` makes every DP rank its own `vllm serve +# --data-parallel-rank` on its own port, so the endpoints file lists 16 endpoints. +# `kv_events` makes every rank publish its KV-cache events to the Endpoint Picker +# (EPP), whose precise prefix-cache producer indexes them; the token producer +# tokenizes each prompt through a worker's render endpoint. srtctl wires the +# event socket, the topics, and the render URL. +# +# Aliases (`llm-d-vllm`) resolve through srtslurm.yaml; see examples/README.md. + +schema: 2 +name: "qwen3-30b-a3b-vllm-llm-d-dp-ranks" + +slurm: + time_limit: "00:30:00" + +model: + path: "hf:Qwen/Qwen3-30B-A3B" + container: "llm-d-vllm" + precision: "bf16" + +resources: + gpu_type: "h100" + gpus_per_node: 8 +frontend: + type: llm-d + enable_multiple_frontends: false + epp_config: + plugins: + # Tokenizes each prompt through a worker's render endpoint (srtctl sets vllm.url and modelName). + - type: token-producer + # Indexes the ranks' KV-cache events (srtctl sets kvEventsConfig's socket). + - type: precise-prefix-cache-producer + parameters: + indexerConfig: + kvBlockIndexConfig: + enableMetrics: true + - type: prefix-cache-scorer + parameters: + prefixMatchInfoProducerName: precise-prefix-cache-producer + - type: queue-scorer + - type: max-score-picker + schedulingProfiles: + - name: default + plugins: + - pluginRef: prefix-cache-scorer + weight: 3 + - pluginRef: queue-scorer + weight: 2 + - pluginRef: max-score-picker + +engine: vllm +roles: + agg: + nodes: 2 + workers: 1 + gpus: 16 + kv_events: true + env: + PYTHONUNBUFFERED: "1" + args: + served-model-name: "Qwen/Qwen3-30B-A3B" + tensor-parallel-size: 1 + data-parallel-size: 16 + data-parallel-external-lb: true + enable-expert-parallel: true + max-model-len: 4096 + +benchmark: + type: "sa-bench" + isl: 128 + osl: 128 + concurrencies: "16x32" diff --git a/mkdocs.yml b/mkdocs.yml index 06020cf42..61776d3aa 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -74,6 +74,7 @@ nav: - SGLang Router: sglang-router.md - vLLM Router: vllm-router.md - Shepherd Model Gateway: smg.md + - llm-d Router: llm-d.md - Benchmarks: - Benchmarks: benchmarks.md - Accuracy Benchmarks: accuracy.md diff --git a/src/srtctl/backends/vllm.py b/src/srtctl/backends/vllm.py index ecf0be3cb..77839b09e 100644 --- a/src/srtctl/backends/vllm.py +++ b/src/srtctl/backends/vllm.py @@ -47,6 +47,7 @@ VLLM_MASTER_PORT_BASE, VLLM_MASTER_PORT_STRIDE, VLLM_SCAN_PORTS, + rank_offset_subscriber_port, ) if TYPE_CHECKING: @@ -433,10 +434,38 @@ def connector_for_mode(self, mode: WorkerMode) -> str | None: mode_connector = self.get_config_for_mode(mode).get("connector") return mode_connector if mode_connector is not None else self.connector + def _effective_connector_for_mode(self, mode: WorkerMode, *, use_default: bool = True) -> str | None: + """Resolve explicit KV-transfer JSON before the named connector configuration. + + Direct aggregate workers skip the engine's P/D default, but still honor + an explicit role configuration. Connector aliases retain their service + and discovery metadata independently of raw payload overrides. + """ + config = self.get_config_for_mode(mode) + for key, value in config.items(): + if normalize_vllm_config_key(key) == "kv-transfer-config" and value is not None: + return json.dumps(dict(value)) if isinstance(value, Mapping) else str(value) + return self.connector_for_mode(mode) if use_default else config.get("connector") + def kv_connector_for_mode(self, mode: WorkerMode) -> KVConnector | None: """The connector table row for a mode; None for no connector or a raw JSON ``--kv-transfer-config``.""" return kv_connector_row(self.connector_for_mode(mode)) + def kv_connector_classes(self, mode: WorkerMode) -> tuple[str, ...]: + """Connector classes from the effective transfer config; empty without a connector. + + Include wrapped classes after ``MultiConnector``. Resolution follows + ``_effective_connector_for_mode``, matching worker command generation. + """ + connector = self._effective_connector_for_mode(mode) + if not connector or connector.lower() in ("null", "none"): + return () + payload = json.loads(_connector_to_kv_transfer_config(connector, mode)) + wrapped = (payload.get("kv_connector_extra_config") or {}).get("connectors") or [] + return tuple( + name for name in (payload.get("kv_connector"), *(inner.get("kv_connector") for inner in wrapped)) if name + ) + def discovers_workers(self) -> bool: """Whether the prefill/decode workers register with the router over its discovery endpoint. @@ -447,7 +476,12 @@ def discovers_workers(self) -> bool: return any(row is not None and row.discovery for row in map(self.kv_connector_for_mode, ("prefill", "decode"))) def kv_transfer_config( - self, mode: WorkerMode, process: Process | None = None, runtime: RuntimeContext | None = None + self, + mode: WorkerMode, + process: Process | None = None, + runtime: RuntimeContext | None = None, + *, + use_default: bool = True, ) -> str | None: """``--kv-transfer-config`` JSON for a worker mode, or None when the mode has no connector. @@ -458,7 +492,7 @@ def kv_transfer_config( this worker's HTTP port and routable IP, and the handshake and notify listeners the allocator reserved for it. """ - connector = self.connector_for_mode(mode) + connector = self._effective_connector_for_mode(mode, use_default=use_default) if not connector or connector.lower() in ("null", "none"): return None row = kv_connector_row(connector) @@ -655,7 +689,7 @@ def allocate_endpoints( ), ) - def _is_dp_mode(self, mode: WorkerMode) -> bool: + def is_dp_mode(self, mode: WorkerMode) -> bool: """Check if this mode uses Data Parallel + Expert Parallel pattern. DP+EP mode is detected when data-parallel-size is set in the mode's config. @@ -664,6 +698,13 @@ def _is_dp_mode(self, mode: WorkerMode) -> bool: dp_size = self._get_dp_size(mode) return dp_size is not None and int(dp_size) > 1 + def is_external_lb(self, mode: WorkerMode) -> bool: + """Whether ``data-parallel-external-lb`` exposes each DP rank as a separate HTTP server.""" + return self.is_dp_mode(mode) and any( + normalize_vllm_config_key(key) == "data-parallel-external-lb" and value + for key, value in self.get_config_for_mode(mode).items() + ) + def _get_dp_size(self, mode: WorkerMode) -> int | None: """Get the data-parallel-size for a mode, or None if not in DP mode.""" config = self.get_config_for_mode(mode) @@ -732,66 +773,10 @@ def _validate_endpoint_parallelism(self, endpoint: Endpoint) -> tuple[int, int]: ) return dp_size, model_parallel_size - def _get_int_flag(self, mode: WorkerMode, name: str, default: int = 1) -> int: - """Read a positive integer CLI flag from the mode config.""" - config = self.get_config_for_mode(mode) - raw = config.get(name) - if raw is None: - raw = config.get(name.replace("-", "_")) - if raw is None: - return default - try: - value = int(raw) - except (TypeError, ValueError) as exc: - raise ValueError(f"{mode} {name}={raw!r} is not an integer") from exc - if value < 1: - raise ValueError(f"{mode} {name}={value} must be >= 1") - return value - - def _tp_size(self, mode: WorkerMode) -> int: - return self._get_int_flag(mode, "tensor-parallel-size", 1) - - def _pp_size(self, mode: WorkerMode) -> int: - return self._get_int_flag(mode, "pipeline-parallel-size", 1) - - def _gpus_per_dp_rank(self, mode: WorkerMode) -> int: - """GPUs consumed by one DP rank: TP × PP.""" - return self._tp_size(mode) * self._pp_size(mode) - - def _local_dp_size(self, mode: WorkerMode, gpu_count: int) -> int: - """DP ranks owned by one process that sees ``gpu_count`` GPUs. - - vLLM world size is TP × DP × PP, so a DP rank occupies TP × PP GPUs. - Those GPUs must fit on the node: a DP rank that spans nodes needs the - non-DP TP launch path (``--nnodes`` / ``--node-rank``), not DP local ranks. - """ - tp = self._tp_size(mode) - pp = self._pp_size(mode) - gpus_per_rank = tp * pp - if gpu_count % gpus_per_rank != 0: - raise ValueError( - f"{mode} tensor-parallel-size={tp} * pipeline-parallel-size={pp} " - f"= {gpus_per_rank} GPUs per DP rank does not divide the " - f"{gpu_count} GPUs allocated on each node" - ) - return gpu_count // gpus_per_rank - - def _validate_dp_world_size(self, mode: WorkerMode, dp_size: int, total_gpus: int) -> None: - """Require TP × DP × PP to match the GPUs allocated to the endpoint.""" - tp = self._tp_size(mode) - pp = self._pp_size(mode) - world = dp_size * tp * pp - if world != total_gpus: - raise ValueError( - f"{mode} data-parallel-size={dp_size} * tensor-parallel-size={tp} " - f"* pipeline-parallel-size={pp} = {world} does not match " - f"the endpoint's {total_gpus} allocated GPUs" - ) - def _dp_rank_gpu_groups(self, mode: WorkerMode, gpu_indices: frozenset[int]) -> list[frozenset[int]]: """Split a node's GPUs into one group per local DP rank.""" - gpus_per_rank = self._gpus_per_dp_rank(mode) - self._local_dp_size(mode, len(gpu_indices)) + gpus_per_rank = self._get_model_parallel_size(mode) + self._get_local_dp_size(mode, len(gpu_indices)) sorted_gpus = sorted(gpu_indices) return [ frozenset(sorted_gpus[offset : offset + gpus_per_rank]) @@ -881,12 +866,23 @@ def endpoints_to_processes( allocator = port_allocator_for(port_allocator, base_sys_port) frontend = get_frontend(frontend_type) if frontend.worker_launch == "direct" and not frontend.expands_node_local_dp: - # One `vllm serve` owns every DP rank of the endpoint, and only the - # leader serves the API, so the standard topology applies. - processes = endpoints_to_processes( - endpoints, port_allocator=allocator, sidecar_grpc=dynamo_sidecar, bootstrap_ports=False - ) - elif not any(self._is_dp_mode(ep.mode) for ep in endpoints): + # Direct serving normally exposes one API per endpoint. External load + # balancing behind a router requires a separate server per DP rank. + processes = [] + for endpoint in endpoints: + if self.is_external_lb(endpoint.mode) and frontend.worker_api_port(endpoint.mode) == "allocated": + processes.extend( + self._dp_per_gpu_endpoints_to_processes( + [endpoint], allocator, sidecar_grpc=dynamo_sidecar, every_rank_serves=True + ) + ) + else: + processes.extend( + endpoints_to_processes( + [endpoint], port_allocator=allocator, sidecar_grpc=dynamo_sidecar, bootstrap_ports=False + ) + ) + elif not any(self.is_dp_mode(ep.mode) for ep in endpoints): # Standard TP mode: one process per node, or one per engine of the # worker under backend.failover (engine 0 plus its shadows). processes = endpoints_to_processes( @@ -928,13 +924,18 @@ def _dp_per_gpu_endpoints_to_processes( allocator: NodePortAllocator, *, sidecar_grpc: bool, + every_rank_serves: bool = False, ) -> list[Process]: - """DP+EP mode with one process per DP rank (TP x PP GPUs each).""" + """DP+EP mode with one process per DP rank (TP x PP x PCP GPUs each). + + Only rank 0 serves the API unless ``every_rank_serves`` (external load + balancing), when every rank gets its own HTTP port. + """ from srtctl.core.topology import Process processes: list[Process] = [] for endpoint in endpoints: - if not self._is_dp_mode(endpoint.mode): + if not self.is_dp_mode(endpoint.mode): # Non-DP endpoints get standard processing (all modes are normally consistent). for node_rank, node in enumerate(endpoint.nodes): is_leader = node_rank == 0 @@ -955,28 +956,27 @@ def _dp_per_gpu_endpoints_to_processes( ) continue - # One process per DP rank. With TP=1 this is one GPU; with TP>1 the - # process owns the TP x PP GPUs for that rank. + # One process per DP rank, owning its TP x PP x PCP GPUs. dp_rank = 0 dp_rpc_port = allocator.next(DP_RPC_PORTS, endpoint.leader_node) - dp_size = self._get_dp_size(endpoint.mode) or (endpoint.total_gpus // self._gpus_per_dp_rank(endpoint.mode)) - self._validate_dp_world_size(endpoint.mode, dp_size, endpoint.total_gpus) + dp_size, _ = self._validate_endpoint_parallelism(endpoint) rank_gpu_groups = self._dp_rank_gpu_groups(endpoint.mode, endpoint.gpu_indices) # vLLM computes actual_port = base + data_parallel_rank, so every DP # rank of the endpoint shares one reserved block. nixl_base_port = allocator.next(NIXL_PORTS, size=dp_size) for node in endpoint.nodes: for rank_gpus in rank_gpu_groups: - is_leader = dp_rank == 0 + serves_api = dp_rank == 0 or every_rank_serves processes.append( Process( node=node, gpu_indices=rank_gpus, sys_port=allocator.next(SYS_PORTS), - http_port=allocator.next(HTTP_PORTS, node) if is_leader else 0, + http_port=allocator.next(HTTP_PORTS, node) if serves_api else 0, endpoint_mode=endpoint.mode, endpoint_index=endpoint.index, node_rank=dp_rank, # dp_rank stored in node_rank for now + dp_rank=dp_rank, kv_events_port=allocator.next(KV_EVENTS_PORTS), nixl_port=nixl_base_port, dp_rpc_port=dp_rpc_port, @@ -1004,7 +1004,7 @@ def _dp_per_node_endpoints_to_processes( processes: list[Process] = [] for endpoint in endpoints: - if not self._is_dp_mode(endpoint.mode): + if not self.is_dp_mode(endpoint.mode): processes.extend( endpoints_to_processes( [endpoint], port_allocator=allocator, sidecar_grpc=sidecar_grpc, bootstrap_ports=False @@ -1163,11 +1163,11 @@ def build_worker_command( # A prefill/decode worker gets its KV connector. An aggregate worker gets # only the one its role names (e.g. lmcache-mp offload), not the P/D default. - role_connector = config.pop("connector", None) - if mode in {"prefill", "decode"} or role_connector is not None: - kv_transfer_config = self.kv_transfer_config(mode, process, runtime) - if kv_transfer_config is not None: - config.setdefault("kv-transfer-config", kv_transfer_config) + kv_transfer_config = self.kv_transfer_config(mode, process, runtime, use_default=mode != "agg") + for key in ("connector", "kv-transfer-config", "kv_transfer_config"): + config.pop(key, None) + if kv_transfer_config is not None: + config["kv-transfer-config"] = kv_transfer_config node_rank = endpoint_nodes.index(process.node) # The worker that is itself the public endpoint may run the alternate @@ -1177,7 +1177,7 @@ def build_worker_command( # Collected as the command is built so the override report below can # name the value srtslurm actually passed for each flag it took over. srtslurm_owned: dict[str, str] = {} - is_dp_mode = self._is_dp_mode(mode) + is_dp_mode = self.is_dp_mode(mode) replica_size = self._get_model_parallel_size(mode) local_gpu_count = len(process.gpu_indices) spans_nodes = replica_size > local_gpu_count @@ -1189,18 +1189,41 @@ def build_worker_command( and not spans_nodes ) + # Behind a router, an external-LB rank is a `vllm serve` of its own (endpoints_to_processes). + is_rank_server = self.is_external_lb(mode) and not binds_public_port and not frontend.expands_node_local_dp + if frontend.expands_node_local_dp and is_dp_mode and self.dp_launch_mode != "per_node": raise ValueError( f"frontend.type: {frontend.type} with data-parallel-size requires backend.dp_launch_mode: per_node" ) - if node_rank == 0 or is_router_local_dp: + if node_rank == 0 or is_router_local_dp or is_rank_server: api_port = runtime.frontend_port if binds_public_port else process.http_port cmd.extend(["--host", "0.0.0.0", "--port", str(api_port)]) srtslurm_owned["host"] = "0.0.0.0" srtslurm_owned["port"] = str(api_port) - if is_router_local_dp: + if is_rank_server: + for key in list(config): + if normalize_vllm_config_key(key) in { + "data-parallel-rank", + "data-parallel-size-local", + "data-parallel-start-rank", + "data-parallel-address", + "data-parallel-rpc-port", + }: + config.pop(key) + cmd.extend( + [ + "--data-parallel-rank", + str(process.dp_rank), + "--data-parallel-address", + leader_ip, + "--data-parallel-rpc-port", + str(process.dp_rpc_port or VLLM_DATA_PARALLEL_RPC_PORT), + ] + ) + elif is_router_local_dp: for key in list(config): if normalize_vllm_config_key(key) in { "data-parallel-size-local", @@ -1271,6 +1294,19 @@ def build_worker_command( device_ids = ",".join(str(i) for i in sorted(process.gpu_indices)) if device_ids: cmd.extend(["--device-ids", device_ids]) + kv_events = self.get_kv_events_config_for_mode(mode) + subscriber = frontend.kv_events_subscriber(process, runtime, served_model_name) if kv_events else None + if kv_events and subscriber is not None: + host, port, topic = subscriber + # Tested vLLM uses the global DP rank for the publisher offset: + # https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/v1/engine/core.py#L1303-L1313 + # https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/v1/core/sched/scheduler.py#L161-L164 + # Explicit TCP addresses connect rather than bind: + # https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/distributed/kv_events.py#L410-L427 + rank = process.dp_rank or 0 + publisher_port = rank_offset_subscriber_port(port, rank) + kv_events = {**kv_events, "endpoint": f"tcp://{host}:{publisher_port}", "topic": topic} + cmd.extend(["--kv-events-config", json.dumps(kv_events)]) cmd.extend(_config_to_cli_args(config)) return cmd @@ -1293,7 +1329,8 @@ def build_worker_command( # KV connector → --kv-transfer-config (dynamo 1.0.0+: --connector was removed). # Pop from config so it doesn't get added again by _config_to_cli_args. - config.pop("connector", None) + for key in ("connector", "kv-transfer-config", "kv_transfer_config"): + config.pop(key, None) kv_transfer_cfg = self.kv_transfer_config(mode, process, runtime) if kv_transfer_cfg is not None: cmd.extend(["--kv-transfer-config", kv_transfer_cfg]) @@ -1307,7 +1344,7 @@ def build_worker_command( cmd.extend(["--device-ids", device_ids]) # Check if this is DP+EP mode (data-parallel-size set) - is_dp_mode = self._is_dp_mode(mode) + is_dp_mode = self.is_dp_mode(mode) if is_dp_mode and self.dp_launch_mode == "per_node": rpc_port_kebab = config.pop("data-parallel-rpc-port", None) rpc_port_snake = config.pop("data_parallel_rpc_port", None) @@ -1443,7 +1480,7 @@ def _build_sidecar_command( ) -> list[str]: """Expose local DP frontends or one frontend for a cross-node replica.""" mode = process.endpoint_mode - is_dp_mode = self._is_dp_mode(mode) + is_dp_mode = self.is_dp_mode(mode) endpoint_nodes = list(dict.fromkeys(candidate.node for candidate in endpoint_processes)) is_multi_node = len(endpoint_nodes) > 1 multi_node_replica = is_multi_node and ( @@ -1586,9 +1623,9 @@ def _build_sidecar_command( if hybrid_lb: command.extend(["--data-parallel-start-rank", str(process.node_rank), "--data-parallel-hybrid-lb"]) - config.pop("connector", None) - has_explicit_kv = "kv-transfer-config" in config or "kv_transfer_config" in config - kv_transfer_cfg = None if has_explicit_kv else self.kv_transfer_config(mode) + for key in ("connector", "kv-transfer-config", "kv_transfer_config"): + config.pop(key, None) + kv_transfer_cfg = self.kv_transfer_config(mode) if kv_transfer_cfg is not None: command.extend(["--kv-transfer-config", kv_transfer_cfg]) diff --git a/src/srtctl/cli/do_sweep.py b/src/srtctl/cli/do_sweep.py index 89bfdd7f3..73ea2a7da 100644 --- a/src/srtctl/cli/do_sweep.py +++ b/src/srtctl/cli/do_sweep.py @@ -33,9 +33,8 @@ TelemetryStageMixin, WorkerStageMixin, ) -from srtctl.core.config import get_srtslurm_setting, load_config +from srtctl.core.config import load_config from srtctl.core.health import wait_for_port -from srtctl.core.launch_plan import configure_launch_plan from srtctl.core.lockfile import write_lockfile from srtctl.core.processes import ( ProcessRegistry, @@ -709,12 +708,6 @@ def _run_post_eval(self, stop_event: threading.Event) -> int: def run(self) -> int: """Run the complete sweep.""" - record_launch_plan = self.config.output.record_launch_plan or bool( - get_srtslurm_setting("record_launch_plan", False) - ) - launch_plan_dir = self.runtime.log_dir / "launch-plan" if record_launch_plan else None - configure_launch_plan(launch_plan_dir, job_id=self.runtime.job_id) - logger.info("Sweep Orchestrator") logger.info("Job ID: %s", self.runtime.job_id) logger.info("Run name: %s", self.runtime.run_name) @@ -726,10 +719,6 @@ def run(self) -> int: logger.info("Pool %s: %s", pool, ", ".join(nodes)) if self.config.profiling.enabled: logger.info("Profiling: %s", self.config.profiling.type) - if launch_plan_dir is not None: - logger.info("Recording realized Slurm launch plan: %s", launch_plan_dir) - # Preserve submission provenance before any runtime stage can fail. - self._copy_config_to_logs() resource_snapshot = record_resource_snapshot(self.config, self.runtime) diff --git a/src/srtctl/cli/mixins/service_stage.py b/src/srtctl/cli/mixins/service_stage.py index 129d485d0..fa0799f1f 100644 --- a/src/srtctl/cli/mixins/service_stage.py +++ b/src/srtctl/cli/mixins/service_stage.py @@ -83,10 +83,11 @@ def service_step_name(service: ServiceConfig, node: str, instances: int, process """Slurm step name (and log stem) for one instance of a service. ``service_`` alone for a single instance, ``service__`` per - node, ``service____`` for an instance attached to a worker. + node, ``service____`` for an instance attached to a worker + (plus ``_dp`` when it attaches to one DP rank of a per-rank worker). """ if process is not None: - return f"service_{service.name}_{process.endpoint_mode}_{process.endpoint_index}_{node}" + return f"service_{service.name}_{process.endpoint_mode}_{process.endpoint_index}_{node}{process.engine_suffix}" suffix = f"_{node}" if instances > 1 else "" return f"service_{service.name}{suffix}" @@ -143,7 +144,8 @@ def service_nodes(self, service: ServiceConfig) -> list[str]: def service_instances(self, service: ServiceConfig) -> list[tuple[str, Process | None]]: """The instances a service launches: ``(node, None)`` per placed node, or under - ``placement.per: worker`` ``(node, process)`` per engine worker on those nodes. + ``placement.per: worker`` ``(node, process)`` per engine worker on those nodes + that the kind attaches to (``ServiceKind.attaches_to``). A worker's instance is attached to its engine 0 process (a worker with shadow engines has several processes on a node; the sidecar serves them all). @@ -155,12 +157,14 @@ def service_instances(self, service: ServiceConfig) -> list[tuple[str, Process | return [(node, None) for node in nodes] where = service.effective_placement order = {node: i for i, node in enumerate(nodes)} + kind = get_service_kind(service.type) attached = [ process for process in self.backend_processes if process.node in order and getattr(process, "engine_id", 0) == 0 and (where == "workers" or process.endpoint_mode == where) + and kind.attaches_to(process) ] attached.sort(key=lambda p: (order[p.node], p.endpoint_index, p.node_rank)) return [(process.node, process) for process in attached] diff --git a/src/srtctl/cli/mixins/worker_stage.py b/src/srtctl/cli/mixins/worker_stage.py index 9e6bb9018..6c32c1545 100644 --- a/src/srtctl/cli/mixins/worker_stage.py +++ b/src/srtctl/cli/mixins/worker_stage.py @@ -39,15 +39,20 @@ WORKER_TERMINATE_TIMEOUT_SECONDS = 30.0 -def worker_step_name(mode: str, index: int, node: str, attempt: int = 0, engine_id: int = 0) -> str: +def worker_step_name( + mode: str, index: int, node: str, attempt: int = 0, engine_id: int = 0, dp_rank: int | None = None +) -> str: """The Slurm step name (and registry name) of a worker step. - ``engine_id`` > 0 is a shadow engine of the worker (vLLM ``engine.failover``) - and adds ``_e``. ``attempt`` is the supervisor's relaunch count; a + ``dp_rank`` names one rank of a per-rank endpoint (``_dp``). ``engine_id`` + > 0 is a shadow engine of the worker (vLLM ``engine.failover``) and adds + ``_e``. ``attempt`` is the supervisor's relaunch count; a relaunched step carries an ``_r`` suffix so ``squeue --steps`` can never confuse it with a lingering step of the life it replaces. """ name = f"{mode}_{index}_{node}" + if dp_rank is not None: + name = f"{name}_dp{dp_rank}" if engine_id: name = f"{name}_e{engine_id}" return f"{name}_r{attempt}" if attempt else name @@ -430,7 +435,9 @@ def __missing__(self, key: str) -> str: endpoint_nodes = {endpoint_process.node for endpoint_process in endpoint_processes} env_to_unset = ["VLLM_PORT"] if backend.type == "vllm" and len(endpoint_nodes) > 1 else None - step_name = worker_step_name(mode, index, process.node, attempt, getattr(process, "engine_id", 0)) + step_name = worker_step_name( + mode, index, process.node, attempt, getattr(process, "engine_id", 0), process.dp_rank + ) proc = start_srun_process( command=cmd, nodelist=[process.node], diff --git a/src/srtctl/core/AGENTS.md b/src/srtctl/core/AGENTS.md index 1652efe92..5be5a16c5 100644 --- a/src/srtctl/core/AGENTS.md +++ b/src/srtctl/core/AGENTS.md @@ -4,7 +4,7 @@ Rules for `src/srtctl/core/` (schema, runtime, topology, processes, health). Sta ## Ports -Fixed ports are constants in `src/srtctl/ports.py`. Every port a worker process binds is a `PortKind` in the same module (`name`, `base`, `stride`, `per_node`) and is handed out by `NodePortAllocator.next(kind, node, size)` in `src/srtctl/core/topology.py`: per-node counters for listeners only bound on that node (`HTTP_PORTS`, `BOOTSTRAP_PORTS`, `DP_RPC_PORTS`, `DIST_INIT_PORTS`, `TOKENSPEED_PORTS`) and global counters for side channels peers address (`SYS_PORTS`, `KV_EVENTS_PORTS`, `NIXL_PORTS`, `KVBM_ZMQ_PORTS`, `NCCL_PORTS`, `VLLM_SCAN_PORTS`, `MORIIO_HANDSHAKE_PORTS`, `MORIIO_NOTIFY_PORTS`, `TRTLLM_DIST_INIT_PORTS`, `SIDECAR_GRPC_PORTS`). `size > 1` reserves a block when the engine adds a rank offset to the port it is given (MoRI-IO adds the local DP and TP rank to its notify base). `endpoints_to_processes` allocates the generic kinds and stores them on `Process`; a backend adds its engine-specific kinds with `dataclasses.replace` on the way out (SGLang `nccl_port` and `dist_init_port`, vLLM `vllm_scan_port` or, for a discovery connector, `moriio_handshake_port` and `moriio_notify_port`, TRT-LLM `trtllm_dist_init_port`, TokenSpeed `tokenspeed_port` and a `dist_init_port` block). `do_sweep` builds the one allocator per job, seeding the sidecar gRPC base from `dynamo.sidecar_port`. +Fixed ports are constants in `src/srtctl/ports.py`. Every port a worker process binds is a `PortKind` in the same module (`name`, `base`, `stride`, `per_node`) and is handed out by `NodePortAllocator.next(kind, node, size)` in `src/srtctl/core/topology.py`: per-node counters for listeners only bound on that node (`HTTP_PORTS`, `BOOTSTRAP_PORTS`, `DP_RPC_PORTS`, `DIST_INIT_PORTS`, `TOKENSPEED_PORTS`) and global counters for side channels peers address (`SYS_PORTS`, `KV_EVENTS_PORTS`, `NIXL_PORTS`, `KVBM_ZMQ_PORTS`, `NCCL_PORTS`, `VLLM_SCAN_PORTS`, `MORIIO_HANDSHAKE_PORTS`, `MORIIO_NOTIFY_PORTS`, `TRTLLM_DIST_INIT_PORTS`, `SIDECAR_GRPC_PORTS`); `PROXY_PORTS` is per node and is allocated after the backends by `worker_processes` (`core/worker_backends.py`), for the routable workers of the modes the frontend proxies (`Frontend.proxied_worker_modes`). `size > 1` reserves a block when the engine adds a rank offset to the port it is given (MoRI-IO adds the local DP and TP rank to its notify base). `endpoints_to_processes` allocates the generic kinds and stores them on `Process`; a backend adds its engine-specific kinds with `dataclasses.replace` on the way out (SGLang `nccl_port` and `dist_init_port`, vLLM `vllm_scan_port` or, for a discovery connector, `moriio_handshake_port` and `moriio_notify_port`, TRT-LLM `trtllm_dist_init_port`, TokenSpeed `tokenspeed_port` and a `dist_init_port` block). `do_sweep` builds the one allocator per job, seeding the sidecar gRPC base from `dynamo.sidecar_port`. Rules that follow: a new listener is a new `PortKind` plus a `Process` field, allocated in the topology builder, never `some_base + (sys_port - DYN_SYSTEM_PORT_BASE)` or any other arithmetic on another port at command-build time. Consumers read the field; a `None` means the topology was built without that kind, and a consumer that needs it raises rather than guessing. A worker's own address is `get_hostname_ip(node, runtime.network_interface)`, not upstream's interface guess. Fixed scan bases such as `VLLM_PORT` exist only to keep co-located `get_open_port()` scans apart; if a connector allocates inside forked children, leave the base unset so the kernel assigns ports. Frontends bind `FRONTEND_PUBLIC_PORT` (8000), or `FRONTEND_INTERNAL_PORT` (8180) behind nginx when `enable_multiple_frontends` is set. Service ports are `ServiceKind` defaults overridden by `options`. `tests/test_port_allocator.py` asserts over every example recipe that no two processes on a node share a port and global kinds never repeat. diff --git a/src/srtctl/core/launch_plan.py b/src/srtctl/core/launch_plan.py deleted file mode 100644 index 8e3d3e85d..000000000 --- a/src/srtctl/core/launch_plan.py +++ /dev/null @@ -1,195 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Persist the realized Slurm launch plan for post-run reproduction.""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import shlex -import threading -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -_SECRET_NAME = re.compile(r"(?:^|_)(?:TOKEN|SECRET|PASSWORD|PASSWD|API_KEY|ACCESS_KEY|PRIVATE_KEY|CREDENTIAL)(?:_|$)") -_SAFE_NAME = re.compile(r"[^A-Za-z0-9_.-]+") - - -def _utc_now() -> str: - return datetime.now(timezone.utc).isoformat() - - -def _is_secret_name(name: str) -> bool: - return bool(_SECRET_NAME.search(name.upper())) - - -def _slug(value: str) -> str: - normalized = _SAFE_NAME.sub("-", value).strip("-.").lower() - return normalized[:80] or "step" - - -def _redacted_command( - command: list[str], - *, - env_to_set: dict[str, str] | None, - srun_export_env: dict[str, str] | None, -) -> tuple[list[str], list[str]]: - """Return a replayable command with secret environment values removed.""" - replay = list(command) - required_secrets: set[str] = set() - - if srun_export_env: - for idx, arg in enumerate(replay): - if not arg.startswith("--export=ALL,"): - continue - exports = [] - for item in arg.removeprefix("--export=ALL,").split(","): - name, separator, value = item.partition("=") - if separator and _is_secret_name(name): - exports.append(name) - required_secrets.add(name) - else: - exports.append(item) - replay[idx] = "--export=ALL," + ",".join(exports) - - if env_to_set: - bash_index = next((idx for idx in range(len(replay) - 1) if replay[idx : idx + 2] == ["bash", "-c"]), None) - if bash_index is not None: - bash_command = replay[bash_index + 2] - for name, value in env_to_set.items(): - if not _is_secret_name(name): - continue - actual = f"export {name}={shlex.quote(value)}" - replacement = f'export {name}="${{{name}:?Set {name} to replay this step}}"' - bash_command = bash_command.replace(actual, replacement) - required_secrets.add(name) - replay[bash_index + 2] = bash_command - - return replay, sorted(required_secrets) - - -class LaunchPlanRecorder: - """Thread-safe recorder for exact, realized ``srun`` invocations.""" - - def __init__(self, directory: Path, *, job_id: str | None = None) -> None: - self.directory = directory - self.directory.mkdir(parents=True, exist_ok=True) - self.job_id = job_id or os.environ.get("SLURM_JOB_ID") or os.environ.get("SLURM_JOBID") - self._lock = threading.Lock() - self._entries: list[dict[str, Any]] = [] - self._write_manifest() - - def record( - self, - command: list[str], - *, - label: str, - output: str | None, - nodelist: list[str] | None, - het_group: int | None, - container_image: str | None, - env_to_set: dict[str, str] | None, - srun_export_env: dict[str, str] | None, - ) -> Path: - replay, required_secrets = _redacted_command( - command, - env_to_set=env_to_set, - srun_export_env=srun_export_env, - ) - with self._lock: - sequence = len(self._entries) + 1 - filename = f"{sequence:03d}-{_slug(label)}.sh" - script_path = self.directory / filename - rendered = self._render_script(replay, required_secrets=required_secrets) - script_path.write_text(rendered) - script_path.chmod(0o750) - - entry = { - "sequence": sequence, - "script": filename, - "recorded_at": _utc_now(), - "output": output, - "nodelist": list(nodelist or []), - "het_group": het_group, - "container_image": container_image, - "required_secret_environment": required_secrets, - "sha256": hashlib.sha256(rendered.encode()).hexdigest(), - } - self._entries.append(entry) - self._write_manifest() - return script_path - - def _render_script(self, command: list[str], *, required_secrets: list[str]) -> str: - lines = [ - "#!/usr/bin/env bash", - "set -euo pipefail", - "", - "# Generated from the exact srun argv used by srt-slurm.", - "# Run inside a compatible active Slurm allocation with the recorded recipe and container available.", - ] - if required_secrets: - lines.append("# Required secret environment (values intentionally omitted): " + ", ".join(required_secrets)) - for name in required_secrets: - lines.append(f': "${{{name}:?Set {name} to replay this step}}"') - lines.extend(["", "exec " + shlex.join(command), ""]) - return "\n".join(lines) - - def _write_manifest(self) -> None: - manifest = { - "schema_version": 1, - "job_id": self.job_id, - "slurm_nodelist": os.environ.get("SLURM_NODELIST"), - "generated_at": _utc_now(), - "replay_scope": "Individual realized Slurm steps; orchestration order and lifecycle remain owned by srtctl.", - "secret_policy": "Secret-like environment values are omitted and named in required_secret_environment.", - "related_artifacts": [ - "../config.yaml", - "../sbatch_script.sh", - "../recipe.lock.yaml", - "../resource_snapshot.json", - ], - "steps": self._entries, - } - temporary = self.directory / ".manifest.json.tmp" - temporary.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") - temporary.replace(self.directory / "manifest.json") - - -_active_recorder: LaunchPlanRecorder | None = None - - -def configure_launch_plan(directory: Path | None, *, job_id: str | None = None) -> LaunchPlanRecorder | None: - """Enable or disable process-wide launch recording for this orchestrator.""" - global _active_recorder - _active_recorder = LaunchPlanRecorder(directory, job_id=job_id) if directory is not None else None - return _active_recorder - - -def record_srun_command( - command: list[str], - *, - label: str, - output: str | None, - nodelist: list[str] | None, - het_group: int | None, - container_image: str | None, - env_to_set: dict[str, str] | None, - srun_export_env: dict[str, str] | None, -) -> Path | None: - """Record a realized command when launch-plan capture is enabled.""" - if _active_recorder is None: - return None - return _active_recorder.record( - command, - label=label, - output=output, - nodelist=nodelist, - het_group=het_group, - container_image=container_image, - env_to_set=env_to_set, - srun_export_env=srun_export_env, - ) diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 24d4987a4..f4f4f2af7 100755 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -411,9 +411,6 @@ class ClusterConfig: # srtctl checkout on the shared filesystem that compute nodes mount at /srtctl-src. Default: this checkout. srtctl_root: str | None = None output_dir: str | None = None # Custom output directory for job logs - # Cluster-wide default for recording exact realized srun commands. Recipes - # can opt in independently with output.record_launch_plan. - record_launch_plan: bool = False # Alias -> path map; a recipe's `model.path` may name an alias instead of a path. model_paths: dict[str, str] | None = None # Alias -> image map, resolved for every container key in a recipe (`model.container`, `roles..container`, ...). @@ -2530,7 +2527,8 @@ class FrontendConfig: Attributes: type: Frontend type - "dynamo" (default); "sglang-router" (SGLang Model - Gateway), "vllm-router", "smg" (Shepherd Model Gateway, any backend), "atomesh", and + Gateway), "vllm-router", "smg" (Shepherd Model Gateway, any backend), "llm-d" + (llm-d Endpoint Picker behind Envoy, vLLM), "atomesh", and "tilert-router" (static routers); "sglang", "vllm", and "trtllm_serve" (direct: the single aggregate worker binds the public port, no router process); "none" (services-only job: no router, no @@ -2564,6 +2562,13 @@ class FrontendConfig: writes this mapping under the top-level ``worker_selection`` key in a generated router policy YAML and passes it to the Dynamo frontend via ``--router-policy-config``. + epp_config: llm-d Endpoint Picker configuration (``EndpointPickerConfig``: + ``plugins``, ``schedulingProfiles``, ...) for ``frontend.type: llm-d``. + srtctl writes it to a file with the ``file-discovery`` plugin and + ``dataLayer.discovery`` added, so the EPP reads the job's workers from + the endpoints file srtctl renders. Omitted, srtctl runs the scorers of + llm-d's no-Kubernetes guide (in prefill and decode profiles for a + prefill/decode job). args: CLI arguments passed to the frontend/router process env: Environment variables for frontend processes container_image: Optional router-specific image. Static routers use the @@ -2583,6 +2588,7 @@ class FrontendConfig: nginx_session_affinity_header: str = "X-Dynamo-Session-ID" nginx_keepalive_timeout: str = "600s" worker_selection: dict[str, Any] | None = None + epp_config: dict[str, Any] | None = None args: dict[str, Any] | None = None env: dict[str, str] | None = None container_image: str | None = None @@ -2602,14 +2608,12 @@ class FrontendConfig: @dataclass(frozen=True) class OutputConfig: - """Output paths and optional reproducibility artifacts.""" + """Output configuration with formattable paths.""" # Directory for job logs and results; a FormattablePath, so `{job_id}` and `$VARS` expand. log_dir: Annotated[FormattablePath, FormattablePathField()] = field( default_factory=lambda: FormattablePath(template="./outputs/{job_id}/logs") ) - # Save the realized `srun` scripts and a manifest under `logs/launch-plan/`. - record_launch_plan: bool = False Schema: ClassVar[type[Schema]] = Schema @@ -3186,16 +3190,18 @@ def _validate_frontend(self) -> None: schema does not know individual frontends. ``none`` is the services-only job and is covered by ``_validate_services_only``. """ - if self.frontend.type == "none": - return from srtctl.frontends import get_frontend, list_frontend_types try: - frontend = get_frontend(self.frontend.type) + frontend = None if self.frontend.type == "none" else get_frontend(self.frontend.type) except ValueError: raise ValidationError( f"Unknown frontend.type {self.frontend.type!r}. Available: {', '.join(list_frontend_types())}" ) from None + if self.frontend.epp_config is not None and (frontend is None or not frontend.accepts_epp_config): + raise ValidationError(f"frontend.epp_config is not supported with frontend.type: {self.frontend.type}") + if frontend is None: + return required = frontend.required_backend incompatible = [ f"{role}={backend.type}" for role, backend in self.active_role_backends() if backend.type != required diff --git a/src/srtctl/core/slurm.py b/src/srtctl/core/slurm.py index 20724e4a2..1aaa4665e 100644 --- a/src/srtctl/core/slurm.py +++ b/src/srtctl/core/slurm.py @@ -20,7 +20,6 @@ from pathlib import Path from .ip_utils import get_node_ip -from .launch_plan import record_srun_command logger = logging.getLogger(__name__) @@ -367,17 +366,6 @@ def start_srun_process( # the srtctl logger to DEBUG when troubleshooting srun arg construction. logger.debug("srun command: %s", shlex.join(srun_cmd)) - record_srun_command( - srun_cmd, - label=Path(output).stem if output else Path(command[0]).name, - output=output, - nodelist=list(nodelist) if nodelist else None, - het_group=het_group, - container_image=str(container_image) if container_image else None, - env_to_set=env_to_set, - srun_export_env=srun_export_env, - ) - # Start the process proc = subprocess.Popen( srun_cmd, diff --git a/src/srtctl/core/supervisor.py b/src/srtctl/core/supervisor.py index 38afdf0ca..2730c48d0 100644 --- a/src/srtctl/core/supervisor.py +++ b/src/srtctl/core/supervisor.py @@ -190,7 +190,10 @@ def track( policy = policy_for(mode) if not policy.enabled: continue - expected = {worker_step_name(mode, index, process.node, engine_id=engine) for process in processes} + expected = { + worker_step_name(mode, index, process.node, engine_id=engine, dp_rank=process.dp_rank) + for process in processes + } mine = sorted(names & expected) if not mine: logger.warning("Worker supervisor: no registered steps found for %s", _label(key)) diff --git a/src/srtctl/core/topology.py b/src/srtctl/core/topology.py index 2ca30a526..16b828ff6 100644 --- a/src/srtctl/core/topology.py +++ b/src/srtctl/core/topology.py @@ -156,12 +156,15 @@ class Process: endpoint_mode: The mode of the parent endpoint endpoint_index: The index of the parent endpoint node_rank: Rank within the endpoint (0 for leader) + dp_rank: The DP rank this process runs when the endpoint is one process per rank + (vLLM ``dp_launch_mode: per_gpu`` or external load balancing); None otherwise engine_id: Which engine of the worker this is. 0 is the one every job has; under ``backend.failover`` (vLLM shadow engine recovery) engines 1.. are the standbys, sharing node, GPUs and node_rank with engine 0 but with their own ports and their own srun step. kvbm_zmq_port: KVBM leader ZMQ pub port (ack is the next port); the leader's is used sidecar_grpc_port: Dynamo sidecar gRPC listener, allocated when the job runs sidecars + proxy_port: Per-worker proxy listener (e.g. llm-d's decode sidecar), when required by the frontend nccl_port: SGLang local TP rendezvous port, one per server process dist_init_port: SGLang multi-node dist-init port; the same value on every process of an endpoint grpc_http_port: HTTP sidecar (/metrics, profiler routes) of an SGLang gRPC-mode leader @@ -189,8 +192,10 @@ class Process: # Inherited from the parent Endpoint when the job is heterogeneous. het_group: int | None = None engine_id: int = 0 + dp_rank: int | None = None kvbm_zmq_port: int | None = None sidecar_grpc_port: int | None = None + proxy_port: int | None = None nccl_port: int | None = None dist_init_port: int | None = None grpc_http_port: int | None = None @@ -207,8 +212,9 @@ def is_leader(self) -> bool: @property def engine_suffix(self) -> str: - """Step-name and log-name suffix that tells a shadow engine apart from engine 0 (``""`` for it).""" - return f"_e{self.engine_id}" if self.engine_id else "" + """Disambiguate colocated processes with optional ``_dp`` and ``_e`` suffixes.""" + rank = f"_dp{self.dp_rank}" if self.dp_rank is not None else "" + return f"{rank}_e{self.engine_id}" if self.engine_id else rank @property def cuda_visible_devices(self) -> str: diff --git a/src/srtctl/core/worker_backends.py b/src/srtctl/core/worker_backends.py index 0ad8bf753..b8c94f91b 100644 --- a/src/srtctl/core/worker_backends.py +++ b/src/srtctl/core/worker_backends.py @@ -11,11 +11,12 @@ from __future__ import annotations from collections.abc import Sequence +from dataclasses import replace from functools import partial from typing import TYPE_CHECKING from srtctl.core.topology import Endpoint, NodePortAllocator, Process, allocate_endpoints -from srtctl.ports import SIDECAR_GRPC_PORTS +from srtctl.ports import PROXY_PORTS, SIDECAR_GRPC_PORTS if TYPE_CHECKING: from srtctl.backends import BackendConfig @@ -65,10 +66,26 @@ def expand(backend: BackendConfig, selected: list[Endpoint]) -> list[Process]: ) if not config.has_role_backends: - return expand(config.backend, endpoints) - processes: list[Process] = [] - for role in ("prefill", "decode", "agg"): - selected = [endpoint for endpoint in endpoints if endpoint.mode == role] - if selected: - processes.extend(expand(config.backend_for_role(role), selected)) - return processes + processes = expand(config.backend, endpoints) + else: + processes = [] + for role in ("prefill", "decode", "agg"): + selected = [endpoint for endpoint in endpoints if endpoint.mode == role] + if selected: + processes.extend(expand(config.backend_for_role(role), selected)) + return with_proxy_ports(config, processes, allocator) + + +def with_proxy_ports(config: SrtConfig, processes: list[Process], allocator: NodePortAllocator) -> list[Process]: + """Allocate proxy ports for routable workers in the frontend's proxied modes.""" + from srtctl.frontends import FRONTEND_NONE, get_frontend + + if config.frontend.type == FRONTEND_NONE: + return processes + proxied = get_frontend(config.frontend.type).proxied_worker_modes(config) + return [ + replace(process, proxy_port=allocator.next(PROXY_PORTS, process.node)) + if process.http_port > 0 and process.endpoint_mode in proxied + else process + for process in processes + ] diff --git a/src/srtctl/frontends/AGENTS.md b/src/srtctl/frontends/AGENTS.md index c648ad7f5..bc4f1e39b 100644 --- a/src/srtctl/frontends/AGENTS.md +++ b/src/srtctl/frontends/AGENTS.md @@ -4,7 +4,7 @@ Rules for `src/srtctl/frontends/`. The Design Rules in the root `CLAUDE.md` appl ## Frontends and readiness -The frontend is the process that owns the public OpenAI port (`FRONTEND_PUBLIC_PORT`) and decides when the job is ready. Implementations live under `src/srtctl/frontends/`, register with `@register_frontend("")`, and are imported from the package `__init__`; `frontend.type` resolves through that registry (`get_frontend`, `list_frontend_types`) and nowhere else. Registered today: `dynamo` (etcd/NATS discovery), `sglang-router`, `vllm-router` and `smg` (static URL routers built on `StaticRouterFrontend`; `smg` pairs with any backend), `trtllm_serve` (direct aggregate or the disaggregated orchestrator), `sglang` and `vllm` (one direct worker owns the port); `none` is the services-only job with no implementation and no gate. +The frontend is the process that owns the public OpenAI port (`FRONTEND_PUBLIC_PORT`) and decides when the job is ready. Implementations live under `src/srtctl/frontends/`, register with `@register_frontend("")`, and are imported from the package `__init__`; `frontend.type` resolves through that registry (`get_frontend`, `list_frontend_types`) and nowhere else. Registered today: `dynamo` (etcd/NATS discovery), `sglang-router`, `vllm-router` and `smg` (static URL routers built on `StaticRouterFrontend`; `smg` pairs with any backend), `llm-d` (the llm-d EPP behind Envoy, on `StaticRouterFrontend`: it writes the workers into an endpoints file instead of a command line, and reaches decode workers through the P/D sidecar service it implies), `trtllm_serve` (direct aggregate or the disaggregated orchestrator), `sglang` and `vllm` (one direct worker owns the port); `none` is the services-only job with no implementation and no gate. `Frontend` hooks: `required_backend` and `validate(config)` (recipe rules, run by `SrtConfig._validate_frontend` at load so dry-run catches them; raise `ValueError` with the user-facing message), `health_endpoint` and `parse_health` (readiness), `get_backend_health_urls` (second gate: every advertised worker URL must answer 200 before traffic), `start_frontends` (launch on `topology.frontend_nodes`, one `ManagedProcess` per node with a `step_name`), `get_frontend_args_list` (`frontend.args` to CLI). The schema knows no individual frontend: pairing and per-type rules come from these two members. @@ -14,6 +14,8 @@ Readiness runs in `BenchmarkStageMixin._wait_for_service_ready`: `wait_for_model The frontend also owns the worker shape, and backends read it instead of comparing names: `worker_launch` (`dynamo` workers register with the Dynamo runtime, `direct` workers are the engine's own server), `worker_api_port(mode)` (`public` when the worker is itself the endpoint and binds `runtime.frontend_port`, `allocated` when a router fronts it on `Process.http_port`), and `expands_node_local_dp` (vLLM Router expands per-node hybrid-LB pools, so the vLLM backend launches one API per node and refuses `per_gpu`). `build_worker_command` still takes `frontend_type` and resolves it through `get_frontend`; a new router mode is therefore one frontend override plus, at most, a new attribute on the protocol. +A frontend that reaches a mode's workers through a per-worker proxy names the modes in `proxied_worker_modes(config)`: `worker_processes` (`core/worker_backends.py`) then allocates `Process.proxy_port` (`PROXY_PORTS`) for every routable worker of those modes, and the service that binds it attaches only to those processes (`ServiceKind.attaches_to`). llm-d's P/D sidecar (`services/llm_d_sidecar.py`) is the user. + Which rank serves what is the frontend's call too, one method per consumer: `worker_metrics_port(process, runtime)` and `worker_metrics_path(backend, mode)` feed the tachometer scrape targets and the AIPerf metrics URLs (a direct worker answers at `backend.prometheus_metrics_path`; `metrics_path` is the router's own) (Dynamo: every rank on its system port; native servers: the leader or each routable pool on its HTTP port), `worker_endpoint_port(process, config, runtime)` feeds the `PREFILL_IPS`-style benchmark env (one per logical worker), `profiling_control_port` and `profiling_control_is_leader_only` feed iteration-triggered nsys control, `direct_endpoint_nodes(processes)` names the nodes whose worker is itself the public endpoint, and `worker_ready_port(process)` is what sequential endpoint start polls. `core/telemetry.py` and the benchmark stage contain no frontend-name checks; `BenchmarkStageMixin.frontend` is `None` for a services-only job. Expected counts come from the frontend too: `health_expectations(config, processes)` returns `(prefill, decode, description)` in the units its endpoint reports. Aggregate workers count as decode; Dynamo counts vLLM DP registrations, vLLM Router counts the ranks it expands each URL into, everything else counts logical workers. diff --git a/src/srtctl/frontends/__init__.py b/src/srtctl/frontends/__init__.py index 4c9f6c9ff..611e15b43 100644 --- a/src/srtctl/frontends/__init__.py +++ b/src/srtctl/frontends/__init__.py @@ -10,6 +10,7 @@ Supported frontend types: - dynamo: Dynamo frontend with NATS/etcd communication +- llm-d: llm-d Endpoint Picker behind Envoy, with the P/D sidecar on decode workers - sglang: Direct sglang.launch_server for a single aggregate worker (no router) - sglang-router: SGLang Model Gateway router in front of static workers - smg: Shepherd Model Gateway in front of any engine's static workers @@ -29,6 +30,7 @@ ) from srtctl.frontends.dynamic_frontend import DynamicFrontend from srtctl.frontends.dynamo import DynamoFrontend +from srtctl.frontends.llm_d import LLMDFrontend from srtctl.frontends.sglang import SGLangRouterFrontend from srtctl.frontends.sglang_direct import SGLangFrontend from srtctl.frontends.smg import SMGFrontend @@ -43,6 +45,7 @@ "DynamicFrontend", "DynamoFrontend", "Frontend", + "LLMDFrontend", "SGLangFrontend", "SGLangRouterFrontend", "SMGFrontend", diff --git a/src/srtctl/frontends/base.py b/src/srtctl/frontends/base.py index 6702f1a63..c552fbe25 100644 --- a/src/srtctl/frontends/base.py +++ b/src/srtctl/frontends/base.py @@ -58,6 +58,9 @@ class Frontend(ABC): #: ``SrtConfig._validate_frontend`` enforces it at config load. required_backend: ClassVar[str | None] = None + #: Whether ``frontend.epp_config`` configures this frontend; ``_validate_frontend`` rejects it otherwise. + accepts_epp_config: ClassVar[bool] = False + #: Role that accepts the public request's model name; None uses decode/agg/prefill order. model_name_role: ClassVar[str | None] = None @@ -180,6 +183,23 @@ def implied_services(self, config: Any) -> list["EffectiveService"]: """Services this frontend needs that the recipe did not name (Dynamo: its discovery plane).""" return [] + def proxied_worker_modes(self, config: Any) -> frozenset[str]: + """Worker modes reached through a per-worker proxy. + + ``worker_processes`` allocates ``Process.proxy_port`` for each routable + worker of these modes; the frontend must imply the service that binds it. + """ + return frozenset() + + def kv_events_subscriber( + self, process: "Process", runtime: "RuntimeContext", model_name: str + ) -> tuple[str, int, str] | None: + """KV-event destination ``(host, port, topic)``, or ``None`` to leave direct workers unconfigured. + + Backends must compensate for any engine-added rank offset to reach this port. + """ + return None + def frontend_metrics_port(self, frontend_args: dict[str, Any] | None) -> int | None: """Port of a Prometheus listener separate from the routing port, or ``None`` when metrics share it.""" return None diff --git a/src/srtctl/frontends/llm_d.py b/src/srtctl/frontends/llm_d.py new file mode 100644 index 000000000..9f487a5eb --- /dev/null +++ b/src/srtctl/frontends/llm_d.py @@ -0,0 +1,530 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""llm-d router frontend (`frontend.type: llm-d`): the Endpoint Picker behind Envoy. + +Envoy accepts HTTP requests and asks the Endpoint Picker (EPP) to select workers +via gRPC ext_proc. It forwards to ``x-gateway-destination-endpoint`` using +ORIGINAL_DST. For P/D, EPP selects a decode sidecar and passes the prefill worker +in ``x-prefiller-host-port``; see ``services/llm_d_sidecar.py``. + +srtctl supplies a fixed endpoint list through EPP's file-discovery plugin, +including one endpoint per external-LB DP rank. Startup waits for worker health +when enabled, writes the configurations, then launches EPP and Envoy. Readiness +requires Envoy's ``/ready`` and EPP's ready-endpoint count, based on worker metrics. +Optional ``roles..kv_events`` feed EPP's precise prefix-cache index. + +Upstream, pinned: llm-d-router v0.10.0 +(https://github.com/llm-d/llm-d-router/tree/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26): +``cmd/epp/runner/runner.go`` (``runWithFileDiscovery``), +``pkg/epp/framework/plugins/datalayer/discovery/file``; the Envoy configuration +follows llm-d's no-Kubernetes guide +(https://github.com/llm-d/llm-d/blob/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment/router/envoy/envoy.yaml). +""" + +from __future__ import annotations + +import copy +import logging +import shlex +import threading +from pathlib import Path +from typing import TYPE_CHECKING, Any, ClassVar + +import requests +import yaml + +from srtctl.core.health import WorkerHealthResult +from srtctl.frontends.base import numactl_prefix, register_frontend +from srtctl.frontends.static_router import StaticRouterFrontend +from srtctl.ports import ( + LLM_D_ENVOY_ADMIN_PORT, + LLM_D_EPP_GRPC_PORT, + LLM_D_EPP_HEALTH_PORT, + LLM_D_EPP_KV_EVENTS_PORT, + LLM_D_EPP_METRICS_PORT, +) +from srtctl.services.config import ServiceConfig, ServicePlacementConfig +from srtctl.services.implicit import EffectiveService, effective_services +from srtctl.services.llm_d_sidecar import LLM_D_SIDECAR_TYPE, sidecar_kv_connector + +if TYPE_CHECKING: + from srtctl.core.processes import ManagedProcess + from srtctl.core.runtime import RuntimeContext + from srtctl.core.topology import Process + +logger = logging.getLogger(__name__) + +# Pool the EPP serves; also the namespace of every endpoint in the endpoints file. +POOL = "srtctl" +DISCOVERY_PLUGIN = "srtctl-file-discovery" +ENDPOINTS_FILE = "llm-d-endpoints.yaml" +EPP_CONFIG_FILE = "llm-d-epp-config.yaml" +ENVOY_CONFIG_FILE = "llm-d-envoy.yaml" +ENVOY_ACCESS_LOG = "llm-d-envoy-access.log" +# The EPP's role label (pkg/epp/framework/plugins/scheduling/filter/bylabel/roles.go): +# prefill-filter and decode-filter select on it; an aggregate worker serves both. +ROLE_LABEL = "llm-d.ai/role" +ROLE_BY_MODE = {"prefill": "prefill", "decode": "decode", "agg": "both"} +# Gauge of endpoints whose metrics the EPP scraped within its staleness window +# (pkg/epp/metrics/llm_d_router_metrics.go). +READY_ENDPOINTS_METRIC = "llm_d_epp_ready_endpoints" +# Precise routing indexes KV-cache events and uses a worker's /v1/*/render endpoints +# to obtain matching token IDs +# (pkg/epp/framework/plugins/requestcontrol/dataproducer/{preciseprefixcache,tokenizer}). +PRECISE_PREFIX_PRODUCER = "precise-prefix-cache-producer" +TOKEN_PRODUCER = "token-producer" +_MANAGED_KV_EVENTS_KEYS = frozenset({"zmqEndpoint", "discoverPods", "podDiscoveryConfig"}) +# Token-producer backends other than vLLM's render endpoints. +_OTHER_TOKENIZERS = frozenset({"estimate", "udsTokenizerConfig"}) +# EPP flags srtctl sets; frontend.args may not repeat them. +_MANAGED_EPP_FLAGS = frozenset( + { + "pool-name", + "pool-namespace", + "config-file", + "config-text", + "grpc-port", + "grpc-health-port", + "metrics-port", + "secure-serving", + } +) + + +def _is_pd(config: Any) -> bool: + topology = config.topology + return topology.num_prefill > 0 and topology.num_decode > 0 + + +def _routed_port(process: Process) -> int: + """Route through the worker's proxy when present, otherwise its HTTP API.""" + return process.proxy_port if process.proxy_port is not None else process.http_port + + +def _container_path(name: str) -> str: + """Container path for a file in ``log_dir``, mounted at ``/logs``.""" + return str(Path("/logs") / name) + + +# Defaults from guides/no-kubernetes-deployment/router/epp/config.yaml. +# In P/D mode, prefill uses all scorers; decode uses only load scorers. +_DEFAULT_SCORERS = { + "queue-scorer": 2, + "kv-cache-utilization-scorer": 2, + "prefix-cache-scorer": 3, + "no-hit-lru-scorer": 2, +} +_DEFAULT_DECODE_SCORERS = ("queue-scorer", "kv-cache-utilization-scorer") + + +def default_epp_config(pd: bool) -> dict[str, Any]: + """Default scheduler when ``frontend.epp_config`` is unset, with separate profiles for P/D.""" + + def profile(name: str, scorers: Any, role_filter: str | None = None) -> dict[str, Any]: + refs = [{"pluginRef": scorer, "weight": _DEFAULT_SCORERS[scorer]} for scorer in scorers] + return {"name": name, "plugins": [{"pluginRef": role_filter}, *refs] if role_filter else refs} + + plugins: list[dict[str, Any]] = [{"type": scorer} for scorer in _DEFAULT_SCORERS] + if not pd: + return {"plugins": plugins, "schedulingProfiles": [profile("default", _DEFAULT_SCORERS)]} + plugins += [ + {"type": "prefill-filter"}, + {"type": "decode-filter"}, + {"type": "always-disagg-pd-decider"}, + {"type": "disagg-profile-handler", "parameters": {"deciders": {"prefill": "always-disagg-pd-decider"}}}, + ] + return { + "plugins": plugins, + "schedulingProfiles": [ + profile("prefill", _DEFAULT_SCORERS, "prefill-filter"), + profile("decode", _DEFAULT_DECODE_SCORERS, "decode-filter"), + ], + } + + +def epp_config_document(epp_config: dict[str, Any] | None, endpoints_path: str) -> dict[str, Any]: + """The recipe's ``EndpointPickerConfig`` with srtctl's file discovery added. + + ``dataLayer.discovery.pluginRef`` is the spelling both v0.10 and v0.11 accept + (v0.11 also reads ``discovery.endpoints.pluginRef``). + """ + document: dict[str, Any] = {"apiVersion": "llm-d.ai/v1alpha1", "kind": "EndpointPickerConfig"} + document.update(copy.deepcopy(epp_config or {})) + document["plugins"] = [ + *(document.get("plugins") or []), + { + "name": DISCOVERY_PLUGIN, + "type": "file-discovery", + # srtctl writes the file once, from the workers it launched. + "parameters": {"path": endpoints_path, "watchFile": False}, + }, + ] + document["dataLayer"] = {**(document.get("dataLayer") or {}), "discovery": {"pluginRef": DISCOVERY_PLUGIN}} + return document + + +def _plugins(epp_config: dict[str, Any] | None, plugin_type: str) -> list[dict[str, Any]]: + return [plugin for plugin in (epp_config or {}).get("plugins") or [] if plugin.get("type") == plugin_type] + + +def _parameters(plugin: dict[str, Any]) -> dict[str, Any]: + return plugin.get("parameters") or {} + + +def _vllm_token_producer(plugin: dict[str, Any]) -> bool: + return not _OTHER_TOKENIZERS & _parameters(plugin).keys() + + +def with_kv_events(document: dict[str, Any], render_url: str, model_name: str) -> dict[str, Any]: + """Configure the KV-event subscriber and the worker URL used for tokenization. + + Workers connect to one EPP socket (``kvEventsConfig.zmqEndpoint``). The + alternative per-pod mode requires a fixed worker port, which collides when + workers share a node. + """ + for plugin in document.get("plugins") or []: + parameters = _parameters(plugin) + if plugin.get("type") == PRECISE_PREFIX_PRODUCER: + kv_events = parameters.get("kvEventsConfig") or {} + parameters["kvEventsConfig"] = { + **kv_events, + "zmqEndpoint": f"tcp://*:{LLM_D_EPP_KV_EVENTS_PORT}", + "discoverPods": False, + } + elif plugin.get("type") == TOKEN_PRODUCER and _vllm_token_producer(plugin): + parameters.setdefault("modelName", model_name) + parameters["vllm"] = {**(parameters.get("vllm") or {}), "url": render_url} + else: + continue + plugin["parameters"] = parameters + return document + + +def envoy_config(listen_port: int, access_log: str) -> str: + """Envoy in front of the EPP: ext_proc to it on localhost, then ORIGINAL_DST to the endpoint it picked.""" + from jinja2 import Environment, FileSystemLoader + + templates = Environment(loader=FileSystemLoader(str(Path(__file__).parent.parent / "templates"))) + return templates.get_template("llm_d_envoy.yaml.j2").render( + admin_port=LLM_D_ENVOY_ADMIN_PORT, + listen_port=listen_port, + access_log=access_log, + epp_port=LLM_D_EPP_GRPC_PORT, + ) + + +def parse_ready_endpoints(metrics_text: str) -> int | None: + """The EPP's ready-endpoint gauge from its Prometheus text, or None before it is published.""" + total: float | None = None + for line in metrics_text.splitlines(): + if line.startswith((f"{READY_ENDPOINTS_METRIC} ", f"{READY_ENDPOINTS_METRIC}{{")): + total = (total or 0.0) + float(line.rsplit(" ", 1)[1]) + return None if total is None else int(total) + + +@register_frontend("llm-d") +class LLMDFrontend(StaticRouterFrontend): + """llm-d Endpoint Picker behind Envoy, in front of direct vLLM workers.""" + + type: ClassVar[str] = "llm-d" + required_backend: ClassVar[str | None] = "vllm" + accepts_epp_config: ClassVar[bool] = True + executable: ClassVar[tuple[str, ...]] = ("epp",) + # The EPP takes prefill/decode from its scheduler configuration, not a flag. + pd_flag: ClassVar[str] = "" + process_name: ClassVar[str] = "llm-d" + # Gate startup on worker health unless health checks are disabled. + wait_for_workers_before_start: ClassVar[bool] = True + + def validate(self, config: Any) -> None: + frontend = config.frontend + self._validate_epp_args(frontend.args) + if frontend.enable_multiple_frontends and config.engine_node_count > 1: + raise ValueError( + "frontend.type: llm-d requires Envoy and the Endpoint Picker on the public endpoint's node " + "for readiness checks; multi-node nginx routing is unsupported, even with " + "num_additional_frontends: 0. Set frontend.enable_multiple_frontends: false" + ) + topology = config.topology + for mode, count in ( + ("prefill", topology.num_prefill), + ("decode", topology.num_decode), + ("agg", topology.num_agg), + ): + if count and config.backend_for_role(mode).is_grpc_mode(mode): + raise ValueError(f"frontend.type: llm-d routes HTTP workers; roles.{mode} serves gRPC") + epp_config = frontend.epp_config or {} + data_layer = epp_config.get("dataLayer") or {} + if "discovery" in data_layer or any( + plugin.get("type") == "file-discovery" or plugin.get("name") == DISCOVERY_PLUGIN + for plugin in epp_config.get("plugins") or [] + ): + raise ValueError( + "frontend.epp_config must not configure endpoint discovery: srtctl adds the file-discovery " + "plugin and dataLayer.discovery for the workers it launches" + ) + self._validate_kv_events(config) + if _is_pd(config): + if frontend.epp_config is not None and not epp_config.get("schedulingProfiles"): + raise ValueError( + "frontend.type: llm-d with prefill and decode workers needs frontend.epp_config with " + "prefill and decode schedulingProfiles and a disaggregation profile handler" + ) + sidecar_kv_connector(config.backend_for_role("decode")) + sidecars = [entry for entry in effective_services(config) if entry.service.type == LLM_D_SIDECAR_TYPE] + if len(sidecars) != 1: + raise ValueError( + "frontend.type: llm-d with prefill and decode workers requires exactly one enabled " + f"{LLM_D_SIDECAR_TYPE} service definition; override the implied service by name" + ) + + def _validate_kv_events(self, config: Any) -> None: + """Workers publish KV-cache events exactly when the EPP has a precise prefix-cache producer to index them.""" + epp_config = config.frontend.epp_config + token_producers = [plugin for plugin in _plugins(epp_config, TOKEN_PRODUCER) if _vllm_token_producer(plugin)] + if any("url" in (_parameters(plugin).get("vllm") or {}) for plugin in token_producers): + raise ValueError(f"frontend.epp_config sets {TOKEN_PRODUCER} vllm.url, which srtctl points at a worker") + topology = config.topology + publishing = [ + mode + for mode, count in ( + ("prefill", topology.num_prefill), + ("decode", topology.num_decode), + ("agg", topology.num_agg), + ) + if count and config.backend_for_role(mode).get_kv_events_config_for_mode(mode) + ] + producers = _plugins(epp_config, PRECISE_PREFIX_PRODUCER) + if not producers: + if publishing: + raise ValueError( + f"roles.{publishing[0]}.kv_events publishes KV-cache events for the EPP to index; " + f"add a {PRECISE_PREFIX_PRODUCER} plugin to frontend.epp_config" + ) + return + if not publishing: + raise ValueError( + f"frontend.epp_config's {PRECISE_PREFIX_PRODUCER} indexes the workers' KV-cache events; " + "set roles..kv_events on the roles it scores" + ) + for producer in producers: + managed = _MANAGED_KV_EVENTS_KEYS & (_parameters(producer).get("kvEventsConfig") or {}).keys() + if managed: + raise ValueError( + f"frontend.epp_config sets {PRECISE_PREFIX_PRODUCER} kvEventsConfig.{', '.join(sorted(managed))}, " + "which srtctl manages: the EPP binds one KV-event socket the workers connect to" + ) + if not token_producers: + raise ValueError( + f"{PRECISE_PREFIX_PRODUCER} needs the engine's token IDs: add a {TOKEN_PRODUCER} plugin " + "to frontend.epp_config (srtctl points it at a worker's render endpoint)" + ) + frontend = config.frontend + if ( + frontend.enable_multiple_frontends and config.engine_node_count > 1 + ) or frontend.placement.location != "head": + raise ValueError( + "roles..kv_events: the workers publish to the EPP on the head node; set " + "frontend.enable_multiple_frontends: false and keep frontend.placement.node: head" + ) + for mode in publishing: + backend = config.backend_for_role(mode) + kv_events = config.roles[mode].kv_events + managed = {"endpoint", "topic"} & kv_events.keys() if isinstance(kv_events, dict) else set() + if managed: + raise ValueError( + f"roles.{mode}.kv_events sets {', '.join(sorted(managed))}, which srtctl manages for llm-d" + ) + if backend.is_dp_mode(mode) and not backend.is_external_lb(mode): + raise ValueError( + f"roles.{mode}.kv_events with data-parallel-size: vLLM publishes every DP rank's events on its " + "own port, so the EPP must route to the ranks themselves; set " + f"roles.{mode}.args.data-parallel-external-lb: true" + ) + + def kv_events_subscriber(self, process: Process, runtime: RuntimeContext, model_name: str) -> tuple[str, int, str]: + """Return the EPP socket and ``kv@@`` topic. + + Use the routed endpoint, including the sidecar port, to match EPP's cache + index keys (pkg/kvevents/engineadapter/vllm_adapter.go). + """ + address = self.resolve_worker_host(process.node, runtime.network_interface) + return runtime.head_node_ip, LLM_D_EPP_KV_EVENTS_PORT, f"kv@{address}:{_routed_port(process)}@{model_name}" + + def worker_metrics_port(self, process: Process, runtime: RuntimeContext) -> int | None: + """Scrape each routable worker directly, including external-LB DP ranks.""" + return process.http_port if process.http_port > 0 else None + + def worker_endpoint_port(self, process: Process, config: Any, runtime: RuntimeContext) -> int | None: + return process.http_port if process.http_port > 0 else None + + def proxied_worker_modes(self, config: Any) -> frozenset[str]: + """Prefill/decode: the router reaches each decode worker through its P/D sidecar.""" + return frozenset({"decode"}) if _is_pd(config) else frozenset() + + def implied_services(self, config: Any) -> list[EffectiveService]: + if not _is_pd(config): + return [] + return [ + EffectiveService( + ServiceConfig( + name=LLM_D_SIDECAR_TYPE, + type=LLM_D_SIDECAR_TYPE, + placement=ServicePlacementConfig(node="decode", per="worker"), + # Nothing on the sidecar's path registers with etcd or NATS. + inherit_discovery_env=False, + ), + implicit=True, + reason="frontend.type llm-d (prefill/decode)", + ) + ] + + def frontend_metrics_port(self, frontend_args: dict[str, Any] | None) -> int | None: + """The EPP serves Prometheus on its own listener.""" + return LLM_D_EPP_METRICS_PORT + + def health_expectations(self, config: Any, processes: list[Process] | None) -> tuple[int, int, str]: + """Count routable processes, including individual external-LB DP ranks.""" + if processes is None: + return super().health_expectations(config, processes) + routable = [process for process in processes if process.http_port > 0] + prefill = sum(process.endpoint_mode == "prefill" for process in routable) + decode = len(routable) - prefill + return prefill, decode, f"{len(routable)} llm-d endpoints" + + def probe_ready( + self, host: str, port: int, expected_prefill: int, expected_decode: int, config: Any + ) -> WorkerHealthResult: + """Envoy's admin ``/ready``, then the EPP's count of endpoints whose metrics it scrapes.""" + envoy = requests.get(f"http://{host}:{LLM_D_ENVOY_ADMIN_PORT}/ready", timeout=5.0) + if envoy.status_code != 200: + return WorkerHealthResult(ready=False, message=f"Envoy /ready returned HTTP {envoy.status_code}") + metrics = requests.get(f"http://{host}:{LLM_D_EPP_METRICS_PORT}/metrics", timeout=5.0) + expected = expected_prefill + expected_decode + ready = parse_ready_endpoints(metrics.text) if metrics.status_code == 200 else None + if ready is None: + return WorkerHealthResult(ready=False, message=f"EPP has not published {READY_ENDPOINTS_METRIC} yet") + return WorkerHealthResult( + ready=ready >= expected, + message=f"llm-d EPP reports {ready}/{expected} endpoints ready", + decode_ready=ready, + decode_expected=expected, + ) + + def endpoints_document( + self, backend_processes: list[Process], network_interface: str | None + ) -> dict[str, list[dict[str, Any]]]: + """The file-discovery endpoints: every routable worker, at its proxy's port when it has one.""" + endpoints = [] + for process in backend_processes: + if process.http_port <= 0: + continue + endpoints.append( + { + "name": f"{process.endpoint_mode}-{process.endpoint_index}-{process.node_rank}", + "namespace": POOL, + "address": self.resolve_worker_host(process.node, network_interface), + "port": str(_routed_port(process)), + "labels": {ROLE_LABEL: ROLE_BY_MODE[process.endpoint_mode]}, + } + ) + return {"endpoints": endpoints} + + def render_url(self, backend_processes: list[Process], network_interface: str | None) -> str: + """Direct worker API for tokenization; prefer the first prefill worker.""" + routable = [process for process in backend_processes if process.http_port > 0] + process = next((p for p in routable if p.endpoint_mode == "prefill"), routable[0]) + return f"http://{self.resolve_worker_host(process.node, network_interface)}:{process.http_port}" + + def _validate_epp_args(self, args: dict[str, Any] | None) -> None: + managed = {str(key).replace("_", "-") for key in (args or {})} & _MANAGED_EPP_FLAGS + if managed: + raise ValueError(f"frontend.args sets {', '.join(sorted(managed))}, which srtctl manages for llm-d") + + def epp_command(self, config: Any, epp_config_path: str) -> list[str]: + user_args = self.get_frontend_args_list(config.frontend.args) + return [ + *self.executable, + f"--pool-name={POOL}", + f"--pool-namespace={POOL}", + f"--config-file={epp_config_path}", + f"--grpc-port={LLM_D_EPP_GRPC_PORT}", + f"--grpc-health-port={LLM_D_EPP_HEALTH_PORT}", + f"--metrics-port={LLM_D_EPP_METRICS_PORT}", + # Envoy dials the EPP over plaintext HTTP/2 on localhost. + "--secure-serving=false", + *user_args, + ] + + def start_frontends( + self, + topology: Any, + runtime: RuntimeContext, + config: Any, + backend: Any, + backend_processes: list[Process], + stop_event: threading.Event | None = None, + ) -> list[ManagedProcess]: + from srtctl.core.processes import FRONTEND_TERMINATE_TIMEOUT_SECONDS, ManagedProcess + + self.wait_for_workers( + self.collect_workers(backend, backend_processes, runtime.network_interface), config, stop_event + ) + + endpoints = self.endpoints_document(backend_processes, runtime.network_interface) + (runtime.log_dir / ENDPOINTS_FILE).write_text(yaml.safe_dump(endpoints, sort_keys=False)) + epp_config = config.frontend.epp_config + epp_document = epp_config_document( + default_epp_config(_is_pd(config)) if epp_config is None else epp_config, _container_path(ENDPOINTS_FILE) + ) + if _plugins(epp_document, PRECISE_PREFIX_PRODUCER) or _plugins(epp_document, TOKEN_PRODUCER): + epp_document = with_kv_events( + epp_document, + self.render_url(backend_processes, runtime.network_interface), + backend.get_served_model_name(runtime.model_path.name), + ) + (runtime.log_dir / EPP_CONFIG_FILE).write_text(yaml.safe_dump(epp_document, sort_keys=False)) + (runtime.log_dir / ENVOY_CONFIG_FILE).write_text( + envoy_config(topology.frontend_port, _container_path(ENVOY_ACCESS_LOG)) + ) + logger.info("llm-d endpoints (%d): %s", len(endpoints["endpoints"]), endpoints["endpoints"]) + + commands = { + "epp": self.epp_command(config, _container_path(EPP_CONFIG_FILE)), + # Avoid hot-restart socket collisions between Envoy instances on the host. + "envoy": ["envoy", "-c", _container_path(ENVOY_CONFIG_FILE), "--disable-hot-restart"], + } + container_image = config.frontend.container_image or str(runtime.container_image) + env = {**runtime.environment, **(config.frontend.env or {})} + processes: list[ManagedProcess] = [] + for idx, node in enumerate(topology.frontend_nodes): + for component, command in commands.items(): + cmd = numactl_prefix(config) + command + step_name = f"{self.process_name}-{component}_{idx}" + log_file = runtime.log_dir / f"{node}_{self.process_name}-{component}_{idx}.out" + logger.info("Starting llm-d %s %d on %s: %s", component, idx, node, shlex.join(cmd)) + proc = self.start_process( + command=cmd, + nodelist=[node], + output=str(log_file), + container_image=container_image, + container_mounts=runtime.container_mounts, + env_to_set=env or None, + het_group=runtime.nodes.het_group_for(node), + step_name=step_name, + srun_options=runtime.srun_options, + ) + processes.append( + ManagedProcess( + name=step_name, + popen=proc, + log_file=log_file, + node=node, + critical=True, + terminate_timeout=FRONTEND_TERMINATE_TIMEOUT_SECONDS, + step_name=step_name, + ) + ) + return processes diff --git a/src/srtctl/frontends/static_router.py b/src/srtctl/frontends/static_router.py index c727796db..bb1f0ef7e 100644 --- a/src/srtctl/frontends/static_router.py +++ b/src/srtctl/frontends/static_router.py @@ -224,6 +224,41 @@ def get_backend_health_urls( """Return extra direct readiness requirements, if any.""" return [] + def wait_for_workers( + self, workers: list[RouterWorker], config: Any, stop_event: threading.Event | None = None + ) -> None: + """Wait for HTTP ``/health`` or gRPC TCP connections; skip when health checks are disabled. + + Raise on cancellation or exhaustion of the recipe's health-check budget. + """ + if config.health_check is None: + return + health_check = config.health_check + timeout = float(health_check.max_attempts * health_check.interval_seconds) + health_urls = [ + f"{worker.url.rstrip('/')}/health" for worker in workers if worker.url.startswith(("http://", "https://")) + ] + grpc_targets = [urlsplit(worker.url) for worker in workers if worker.url.startswith("grpc://")] + logger.info("Waiting for %d advertised backend endpoints before starting %s", len(workers), self.type) + deadline = time.monotonic() + timeout + ready = all( + wait_for_port( + str(target.hostname), + int(target.port or 0), + timeout=max(deadline - time.monotonic(), 0.0), + stop_event=stop_event, + ) + for target in grpc_targets + ) and wait_for_http_endpoints( + health_urls, + poll_interval=float(health_check.interval_seconds), + timeout=max(deadline - time.monotonic(), 0.0), + report_every=60.0, + stop_event=stop_event, + ) + if not ready: + raise RuntimeError(f"Advertised backend endpoints did not become ready before {self.type} startup") + def discovers_workers(self, backend: Any) -> bool: """Whether the router learns its workers by registration instead of from its command line. @@ -283,35 +318,8 @@ def start_frontends( ) workers = self.collect_workers(backend, backend_processes, runtime.network_interface) - if self.wait_for_workers_before_start and config.health_check is not None: - health_check = config.health_check - timeout = float(health_check.max_attempts * health_check.interval_seconds) - health_urls = [ - f"{worker.url.rstrip('/')}/health" - for worker in workers - if worker.url.startswith(("http://", "https://")) - ] - # A gRPC worker serves no HTTP /health; its port accepting connections is the signal. - grpc_targets = [urlsplit(worker.url) for worker in workers if worker.url.startswith("grpc://")] - logger.info("Waiting for %d advertised backend endpoints before starting %s", len(workers), self.type) - deadline = time.monotonic() + timeout - ready = all( - wait_for_port( - str(target.hostname), - int(target.port or 0), - timeout=max(deadline - time.monotonic(), 0.0), - stop_event=stop_event, - ) - for target in grpc_targets - ) and wait_for_http_endpoints( - health_urls, - poll_interval=float(health_check.interval_seconds), - timeout=max(deadline - time.monotonic(), 0.0), - report_every=60.0, - stop_event=stop_event, - ) - if not ready: - raise RuntimeError(f"Advertised backend endpoints did not become ready before {self.type} startup") + if self.wait_for_workers_before_start: + self.wait_for_workers(workers, config, stop_event) processes: list[ManagedProcess] = [] for idx, node in enumerate(topology.frontend_nodes): diff --git a/src/srtctl/frontends/vllm_router.py b/src/srtctl/frontends/vllm_router.py index e428943aa..8f5d77eca 100644 --- a/src/srtctl/frontends/vllm_router.py +++ b/src/srtctl/frontends/vllm_router.py @@ -92,7 +92,7 @@ def validate(self, config: Any) -> None: if gpu_count <= 0: continue backend = config.backend_for_role(mode) - if not backend._is_dp_mode(mode): + if not backend.is_dp_mode(mode): expansion_by_mode[mode] = 1 continue if backend.dp_launch_mode != "per_node": diff --git a/src/srtctl/ports.py b/src/srtctl/ports.py index 6106f2d4e..cbfde1a48 100644 --- a/src/srtctl/ports.py +++ b/src/srtctl/ports.py @@ -42,6 +42,16 @@ # passed as --prometheus-port so srtctl, not upstream's default, owns it. SMG_METRICS_PORT = 29000 +# llm-d router-node listeners, passed explicitly; Envoy admin serves /ready. +LLM_D_EPP_GRPC_PORT = 9002 +LLM_D_EPP_HEALTH_PORT = 9003 +LLM_D_EPP_METRICS_PORT = 9090 +LLM_D_ENVOY_ADMIN_PORT = 9901 +# EPP binds this ZMQ subscriber; all KV-event publishers connect to it. +LLM_D_EPP_KV_EVENTS_PORT = 5557 +# Per-worker proxy listeners, including llm-d decode sidecars; see PROXY_PORTS. +WORKER_PROXY_PORT_BASE = 9600 + # TRT-LLM torch.distributed bootstrap, one port per MPI endpoint. TRTLLM_DIST_INIT_PORT_BASE = 29500 @@ -96,6 +106,15 @@ DYNAMO_SIDECAR_GRPC_PORT = 50051 +def rank_offset_subscriber_port(port: int, rank: int) -> int: + """Undo vLLM's global DP-rank offset to reach a shared KV-event subscriber. + + Tested image's offset_endpoint_port: + https://github.com/vllm-project/vllm/blob/ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9/vllm/distributed/kv_events.py#L495-L523 + """ + return port - rank + + @dataclass(frozen=True) class PortKind: """One kind of listener a worker process binds, allocated by ``NodePortAllocator``. @@ -125,6 +144,8 @@ class PortKind: # KVBM leader ZMQ pair: pub at the port, ack at the port + 1. KVBM_ZMQ_PORTS = PortKind("kvbm_zmq", KVBM_ZMQ_PORT_BASE, 2) SIDECAR_GRPC_PORTS = PortKind("sidecar_grpc", DYNAMO_SIDECAR_GRPC_PORT) +# Allocated on each worker's node for Frontend.proxied_worker_modes. +PROXY_PORTS = PortKind("proxy", WORKER_PROXY_PORT_BASE, per_node=True) # Engine-specific: the backend allocates these for its own processes. NCCL_PORTS = PortKind("nccl", SGLANG_NCCL_PORT_BASE) DIST_INIT_PORTS = PortKind("dist_init", SGLANG_DIST_INIT_PORT_BASE, per_node=True) diff --git a/src/srtctl/services/__init__.py b/src/srtctl/services/__init__.py index 81e220712..5e9c82103 100644 --- a/src/srtctl/services/__init__.py +++ b/src/srtctl/services/__init__.py @@ -4,7 +4,17 @@ """The top-level ``services:`` block: user-declared long-running processes launched next to the job.""" # Import kinds to trigger registration. -from srtctl.services import exporters, generic, gms, infra, lmcache_server, mooncake_master, mooncake_store, ray +from srtctl.services import ( + exporters, + generic, + gms, + infra, + llm_d_sidecar, + lmcache_server, + mooncake_master, + mooncake_store, + ray, +) from srtctl.services.config import ( SERVICE_PLACEMENTS, SERVICE_STARTS, @@ -42,6 +52,7 @@ "gms", "infra", "list_service_types", + "llm_d_sidecar", "lmcache_server", "mooncake_master", "mooncake_store", diff --git a/src/srtctl/services/llm_d_sidecar.py b/src/srtctl/services/llm_d_sidecar.py new file mode 100644 index 000000000..87b56be8d --- /dev/null +++ b/src/srtctl/services/llm_d_sidecar.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``type: llm-d-sidecar``: llm-d's P/D sidecar in front of every routable decode worker. + +The llm-d frontend implies this service for P/D jobs. The sidecar calls the +prefill worker named in ``x-prefiller-host-port``, then passes the returned +``kv_transfer_params`` to its local vLLM, which pulls the prefill KV cache. +It listens on ``Process.proxy_port`` and forwards other routes, including +``/metrics``, to the worker's HTTP port. + +Upstream: llm-d-router ``cmd/pd-sidecar`` and ``pkg/sidecar/proxy`` at v0.10.0 +(https://github.com/llm-d/llm-d-router/tree/71f4f0999f95b96c49a9d0c4afbd18dfdb943c26/pkg/sidecar/proxy). +Its ``GET /health`` is independent of vLLM, allowing it to start before workers. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from marshmallow import ValidationError + +from srtctl.services.config import HttpProbe, ServiceReadinessConfig +from srtctl.services.registry import ServiceKind, ServiceLaunchContext, register_service + +if TYPE_CHECKING: + from srtctl.core.schema import SrtConfig + from srtctl.core.topology import Process + from srtctl.services.config import ServiceConfig + +LLM_D_SIDECAR_TYPE = "llm-d-sidecar" + +# The sidecar's --kv-connector protocol for the vLLM connector class the decode workers run. +# Upstream's table of protocols: pkg/sidecar/proxy/options.go (supportedKVConnectors). +SIDECAR_KV_CONNECTORS: dict[str, str] = { + "NixlConnector": "nixlv2", +} + +# Argv shown by dry-run, where there is no worker to read the ports from. +_PREVIEW_PORTS = ["--port=", "--model-server-port="] +_MANAGED_FLAGS = frozenset({"--port", "--model-server-port", "--kv-connector", "--secure-proxy"}) + + +def sidecar_kv_connector(backend: Any) -> str: + """Select the decode connector's sidecar protocol, or raise ``ValueError``. + + For ``MultiConnector``, use the first supported wrapped connector. + """ + from srtctl.backends.vllm import VLLMBackend + + # Only vLLM's KV-transfer protocol is supported. + connectors = backend.kv_connector_classes("decode") if isinstance(backend, VLLMBackend) else () + protocol = next((SIDECAR_KV_CONNECTORS[name] for name in connectors if name in SIDECAR_KV_CONNECTORS), None) + if protocol is None: + supported = ", ".join(f"{name} ({protocol})" for name, protocol in SIDECAR_KV_CONNECTORS.items()) + raise ValueError( + f"the llm-d P/D sidecar has no protocol for the {backend.type} decode KV connector " + f"{' > '.join(connectors) or None!r}; supported (vLLM, alone or in a MultiConnector): {supported}" + ) + return protocol + + +@register_service(LLM_D_SIDECAR_TYPE) +class LLMDSidecarService(ServiceKind): + """llm-d P/D sidecar, one per routable decode worker; implied by the llm-d frontend.""" + + builds_command = True + default_command = ("pd-sidecar",) + default_start = "before_workers" + default_critical = True + default_placement = "decode" + default_per = "worker" + + def validate(self, service: ServiceConfig, config: SrtConfig) -> None: + from srtctl.frontends import FRONTEND_NONE, get_frontend + + conflicts = { + arg.split("=", 1)[0] + for arg in [*(service.command or []), *service.args] + if arg.startswith("--") and arg.split("=", 1)[0] in _MANAGED_FLAGS + } + if conflicts: + raise ValidationError( + f"services[{service.name}].command/args sets {', '.join(sorted(conflicts))}, " + "which srtctl manages for the worker's ports and KV-transfer protocol" + ) + proxied = ( + frozenset() + if config.frontend.type == FRONTEND_NONE + else get_frontend(config.frontend.type).proxied_worker_modes(config) + ) + if service.effective_per != "worker" or service.effective_placement not in proxied: + raise ValidationError( + f"services[{service.name}] (type {LLM_D_SIDECAR_TYPE}) fronts the workers the frontend proxies " + f"({', '.join(sorted(proxied)) or 'none for this recipe'}); set placement.node to one of them " + "and placement.per: worker" + ) + + def container_fallback(self, config: SrtConfig) -> str | None: + """Use the frontend image, which also contains ``pd-sidecar``.""" + return config.frontend.container_image + + def attaches_to(self, process: Process) -> bool: + return process.proxy_port is not None + + def build_command(self, service: ServiceConfig, ctx: ServiceLaunchContext) -> list[str]: + command = list(service.command) if service.command is not None else list(self.default_command or ()) + if ctx.process is None or ctx.config is None or ctx.process.proxy_port is None: + managed = [*_PREVIEW_PORTS, "--kv-connector="] + else: + connector = sidecar_kv_connector(ctx.config.backend_for_role(ctx.process.endpoint_mode)) + managed = [ + f"--port={ctx.process.proxy_port}", + f"--model-server-port={ctx.process.http_port}", + f"--kv-connector={connector}", + ] + # The router reaches the sidecar over plain HTTP inside the job. + return [*command, *managed, "--secure-proxy=false", *service.args] + + def readiness(self, service: ServiceConfig, ctx: ServiceLaunchContext) -> ServiceReadinessConfig | None: + if ctx.process is None or ctx.process.proxy_port is None: + return None + return ServiceReadinessConfig( + http=HttpProbe(port=ctx.process.proxy_port, path="/health"), + timeout_seconds=self.default_readiness_timeout, + ) diff --git a/src/srtctl/services/registry.py b/src/srtctl/services/registry.py index 9722e105e..8332eed3f 100644 --- a/src/srtctl/services/registry.py +++ b/src/srtctl/services/registry.py @@ -105,6 +105,10 @@ def template_vars(self) -> dict[str, str]: "worker_gpu_count": str(len(self.process.gpu_indices)), } ) + if self.process.http_port > 0: + values["worker_http_port"] = str(self.process.http_port) + if self.process.proxy_port is not None: + values["worker_proxy_port"] = str(self.process.proxy_port) return values @@ -196,6 +200,14 @@ def wait_fleet_ready(self, service: ServiceConfig, runtime: RuntimeContext, proc also needs the head to report every member before the client starts. """ + def attaches_to(self, process: Process) -> bool: + """Filter eligible processes for ``placement.per: worker`` services. + + By default, attach to all engine-0 processes, including followers. + API proxies override this to exclude processes without a listener. + """ + return True + def metrics(self, service: ServiceConfig) -> list[ServiceMetricsConfig]: """The Prometheus endpoints this service serves; empty when it publishes none. diff --git a/src/srtctl/templates/llm_d_envoy.yaml.j2 b/src/srtctl/templates/llm_d_envoy.yaml.j2 new file mode 100644 index 000000000..98c647870 --- /dev/null +++ b/src/srtctl/templates/llm_d_envoy.yaml.j2 @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Envoy configuration based on llm-d's no-Kubernetes guide: +# https://github.com/llm-d/llm-d/blob/7fb84b0adf8e1d41eb2cef105fc1aafaf5a9b64f/guides/no-kubernetes-deployment/router/envoy/envoy.yaml +# Allow remote /ready probes and long-running streams; log selected endpoints. + +admin: + address: + socket_address: {address: 0.0.0.0, port_value: {{ admin_port }}} + +static_resources: + listeners: + - name: llm-d + address: + socket_address: {address: 0.0.0.0, port_value: {{ listen_port }}} + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: llm-d + stream_idle_timeout: 0s + request_timeout: 0s + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: {{ access_log }} + log_format: + text_format_source: + inline_string: "[%START_TIME%] %REQ(:METHOD)% %REQ(:PATH)% status=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% upstream=%UPSTREAM_HOST% prefiller=%REQ(x-prefiller-host-port)% duration=%DURATION%ms\n" + route_config: + name: llm-d + virtual_hosts: + - name: llm-d + domains: ["*"] + routes: + # Scrape workers and EPP directly; load balancing would mix workers' counters. + - match: {path: "/metrics"} + direct_response: {status: 404} + typed_per_filter_config: + envoy.filters.http.ext_proc: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExtProcPerRoute + disabled: true + - match: {prefix: "/"} + route: {cluster: original_destination_cluster, timeout: 0s} + http_filters: + - name: envoy.filters.http.ext_proc + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor + grpc_service: + envoy_grpc: {cluster_name: ext_proc} + timeout: 10s + processing_mode: + request_header_mode: SEND + response_header_mode: SEND + request_body_mode: FULL_DUPLEX_STREAMED + response_body_mode: FULL_DUPLEX_STREAMED + request_trailer_mode: SEND + response_trailer_mode: SEND + message_timeout: 1000s + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppress_envoy_headers: true + + clusters: + # The worker the EPP picked, named by its x-gateway-destination-endpoint header. + - name: original_destination_cluster + type: ORIGINAL_DST + connect_timeout: 1000s + lb_policy: CLUSTER_PROVIDED + # Each request holds a worker connection and EPP stream; raise the default 1024 concurrency cap. + circuit_breakers: + thresholds: + - {max_connections: 40000, max_pending_requests: 40000, max_requests: 40000} + original_dst_lb_config: + use_http_header: true + http_header_name: x-gateway-destination-endpoint + + - name: ext_proc + type: STATIC + connect_timeout: 86400s + lb_policy: LEAST_REQUEST + circuit_breakers: + thresholds: + - {max_connections: 40000, max_pending_requests: 40000, max_requests: 40000, max_retries: 1024} + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: ext_proc + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: {address: 127.0.0.1, port_value: {{ epp_port }}} diff --git a/srtslurm.yaml.example b/srtslurm.yaml.example index cb305a719..932131f3c 100644 --- a/srtslurm.yaml.example +++ b/srtslurm.yaml.example @@ -76,12 +76,6 @@ use_segment_sbatch_directive: true # Default: true # Set to true if your cluster requires --exclusive to be set. use_exclusive_sbatch_directive: false # Default: false -# Reproducibility artifacts (optional) -# Record every realized srun invocation as an executable, secret-redacted script -# under outputs//logs/launch-plan/. Individual recipes can also enable -# this with output.record_launch_plan: true. -record_launch_plan: false - # Container aliases (optional; a recipe may also name the image directly in model.container) containers: sglang-latest: "/shared/containers/sglang-v0.4.sqsh" diff --git a/tests/snapshots/launch/vllm__llm-d-agg.txt b/tests/snapshots/launch/vllm__llm-d-agg.txt new file mode 100644 index 000000000..3d14205a2 --- /dev/null +++ b/tests/snapshots/launch/vllm__llm-d-agg.txt @@ -0,0 +1,190 @@ +# examples/vllm/llm-d-agg.yaml +# exit_code: 0 +# srun calls: 12 + +## agg_0_node-01 +nodelist: node-01 +output: /logs/node-01_agg_w0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5200 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20000 +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6100 --device-ids 0 --gpu-memory-utilization 0.5 --max-model-len 4096 --served-model-name Qwen/Qwen3-0.6B --tensor-parallel-size 1 + +## agg_1_node-01 +nodelist: node-01 +output: /logs/node-01_agg_w1.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5201 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5401 + VLLM_PORT: 20050 +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6132 --device-ids 1 --gpu-memory-utilization 0.5 --max-model-len 4096 --served-model-name Qwen/Qwen3-0.6B --tensor-parallel-size 1 + +## llm-d-epp_0 +nodelist: node-01 +output: /logs/node-01_llm-d-epp_0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +command: | + epp --pool-name=srtctl --pool-namespace=srtctl --config-file=/logs/llm-d-epp-config.yaml --grpc-port=9002 --grpc-health-port=9003 --metrics-port=9090 --secure-serving=false + +## llm-d-envoy_0 +nodelist: node-01 +output: /logs/node-01_llm-d-envoy_0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +command: | + envoy -c /logs/llm-d-envoy.yaml --disable-hot-restart + +## service_dcgm-exporter_node-01 +nodelist: node-01 +output: /logs/service_dcgm-exporter_node-01.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-02 +nodelist: node-02 +output: /logs/service_dcgm-exporter_node-02.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-03 +nodelist: node-03 +output: /logs/service_dcgm-exporter_node-03.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_node-exporter_node-01 +nodelist: node-01 +output: /logs/service_node-exporter_node-01.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-02 +nodelist: node-02 +output: /logs/service_node-exporter_node-02.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-03 +nodelist: node-03 +output: /logs/service_node-exporter_node-03.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## tachometer +nodelist: node-01 +output: /logs/tachometer.out +srun_export_env: + POLARS_MAX_THREADS: 4 +use_bash_wrapper: False +command: | + tachometer-scraper --config /logs/tachometer_config.toml --local-dir /logs/tachometer/local --sync-interval 120 + +## +nodelist: node-01 +output: /logs/benchmark.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + SRTCTL_FRONTEND_TYPE: llm-d + SRT_FRONTEND_HOST: 127.0.0.1 + SRT_FRONTEND_PORT: 8000 +command: | + bash /srtctl-benchmarks/sa-bench/bench.sh http://localhost:8000 128 128 4x8 inf /model Qwen/Qwen3-0.6B false 8 0 0 0.8 10 2 '' true random '' false diff --git a/tests/snapshots/launch/vllm__llm-d-disagg.txt b/tests/snapshots/launch/vllm__llm-d-disagg.txt new file mode 100644 index 000000000..cf184fab1 --- /dev/null +++ b/tests/snapshots/launch/vllm__llm-d-disagg.txt @@ -0,0 +1,205 @@ +# examples/vllm/llm-d-disagg.yaml +# exit_code: 0 +# srun calls: 13 + +## service_llm-d-sidecar_decode_0_node-02 +nodelist: node-02 +output: /logs/service_llm-d-sidecar_decode_0_node-02.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + CUDA_VISIBLE_DEVICES: 0 +use_bash_wrapper: True +command: | + pd-sidecar --port=9600 --model-server-port=6100 --kv-connector=nixlv2 --secure-proxy=false + +## prefill_0_node-01 +nodelist: node-01 +output: /logs/node-01_prefill_w0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5200 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20000 +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6100 --device-ids 0 --gpu-memory-utilization 0.5 --kv-transfer-config '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' --max-model-len 4096 --served-model-name Qwen/Qwen3-0.6B --tensor-parallel-size 1 + +## decode_0_node-02 +nodelist: node-02 +output: /logs/node-02_decode_w0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5201 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5401 + VLLM_PORT: 20050 +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6100 --device-ids 0 --gpu-memory-utilization 0.5 --kv-transfer-config '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' --max-model-len 4096 --served-model-name Qwen/Qwen3-0.6B --tensor-parallel-size 1 + +## llm-d-epp_0 +nodelist: node-01 +output: /logs/node-01_llm-d-epp_0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +command: | + epp --pool-name=srtctl --pool-namespace=srtctl --config-file=/logs/llm-d-epp-config.yaml --grpc-port=9002 --grpc-health-port=9003 --metrics-port=9090 --secure-serving=false + +## llm-d-envoy_0 +nodelist: node-01 +output: /logs/node-01_llm-d-envoy_0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +command: | + envoy -c /logs/llm-d-envoy.yaml --disable-hot-restart + +## service_dcgm-exporter_node-01 +nodelist: node-01 +output: /logs/service_dcgm-exporter_node-01.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-02 +nodelist: node-02 +output: /logs/service_dcgm-exporter_node-02.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-03 +nodelist: node-03 +output: /logs/service_dcgm-exporter_node-03.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_node-exporter_node-01 +nodelist: node-01 +output: /logs/service_node-exporter_node-01.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-02 +nodelist: node-02 +output: /logs/service_node-exporter_node-02.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-03 +nodelist: node-03 +output: /logs/service_node-exporter_node-03.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## tachometer +nodelist: node-01 +output: /logs/tachometer.out +srun_export_env: + POLARS_MAX_THREADS: 4 +use_bash_wrapper: False +command: | + tachometer-scraper --config /logs/tachometer_config.toml --local-dir /logs/tachometer/local --sync-interval 120 + +## +nodelist: node-01 +output: /logs/benchmark.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + SRTCTL_FRONTEND_TYPE: llm-d + SRT_FRONTEND_HOST: 127.0.0.1 + SRT_FRONTEND_PORT: 8000 +command: | + bash /srtctl-benchmarks/sa-bench/bench.sh http://localhost:8000 128 128 4x8 inf /model Qwen/Qwen3-0.6B true 2 1 1 0.8 10 2 '' true random '' false diff --git a/tests/snapshots/launch/vllm__llm-d-dp-ranks.txt b/tests/snapshots/launch/vllm__llm-d-dp-ranks.txt new file mode 100644 index 000000000..fc0355488 --- /dev/null +++ b/tests/snapshots/launch/vllm__llm-d-dp-ranks.txt @@ -0,0 +1,572 @@ +# examples/vllm/llm-d-dp-ranks.yaml +# exit_code: 0 +# srun calls: 28 + +## agg_0_node-01_dp0 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5200 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20000 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6100 --data-parallel-rank 0 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 0 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6100@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5557"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp1 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp1.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5201 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20050 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6132 --data-parallel-rank 1 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 1 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6132@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5556"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp2 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp2.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5202 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20100 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6164 --data-parallel-rank 2 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 2 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6164@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5555"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp3 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp3.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5203 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20150 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6196 --data-parallel-rank 3 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 3 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6196@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5554"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp4 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp4.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5204 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20200 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6228 --data-parallel-rank 4 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 4 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6228@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5553"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp5 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp5.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5205 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20250 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6260 --data-parallel-rank 5 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 5 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6260@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5552"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp6 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp6.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5206 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20300 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6292 --data-parallel-rank 6 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 6 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6292@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5551"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-01_dp7 +nodelist: node-01 +output: /logs/node-01_agg_w0_dp7.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5207 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20350 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6324 --data-parallel-rank 7 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 7 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6324@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5550"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp8 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp8.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5208 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20400 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6100 --data-parallel-rank 8 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 0 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6100@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5549"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp9 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp9.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5209 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20450 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6132 --data-parallel-rank 9 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 1 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6132@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5548"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp10 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp10.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5210 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20500 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6164 --data-parallel-rank 10 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 2 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6164@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5547"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp11 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp11.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5211 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20550 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6196 --data-parallel-rank 11 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 3 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6196@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5546"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp12 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp12.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5212 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20600 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6228 --data-parallel-rank 12 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 4 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6228@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5545"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp13 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp13.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5213 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20650 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6260 --data-parallel-rank 13 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 5 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6260@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5544"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp14 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp14.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5214 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20700 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6292 --data-parallel-rank 14 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 6 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6292@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5543"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## agg_0_node-02_dp15 +nodelist: node-02 +output: /logs/node-02_agg_w0_dp15.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + DYN_VLLM_KV_EVENT_PORT: 5215 + HEAD_NODE_IP: 127.0.0.1 + PYTHONUNBUFFERED: 1 + VLLM_NIXL_SIDE_CHANNEL_HOST: 127.0.0.1 + VLLM_NIXL_SIDE_CHANNEL_PORT: 5400 + VLLM_PORT: 20750 +env_to_unset: VLLM_PORT +bash_preamble: | + ( python3 <(cat <<'__FINGERPRINT_EOF__' + + __FINGERPRINT_EOF__ + ) || true ) +command: | + vllm serve /model --host 0.0.0.0 --port 6324 --data-parallel-rank 15 --data-parallel-address 127.0.0.1 --data-parallel-rpc-port 8400 --device-ids 7 --kv-events-config '{"publisher": "zmq", "topic": "kv@127.0.0.1:6324@Qwen/Qwen3-30B-A3B", "enable_kv_cache_events": true, "endpoint": "tcp://127.0.0.1:5542"}' --data-parallel-external-lb --data-parallel-size 16 --enable-expert-parallel --max-model-len 4096 --served-model-name Qwen/Qwen3-30B-A3B --tensor-parallel-size 1 + +## llm-d-epp_0 +nodelist: node-01 +output: /logs/node-01_llm-d-epp_0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +command: | + epp --pool-name=srtctl --pool-namespace=srtctl --config-file=/logs/llm-d-epp-config.yaml --grpc-port=9002 --grpc-health-port=9003 --metrics-port=9090 --secure-serving=false + +## llm-d-envoy_0 +nodelist: node-01 +output: /logs/node-01_llm-d-envoy_0.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +command: | + envoy -c /logs/llm-d-envoy.yaml --disable-hot-restart + +## service_dcgm-exporter_node-01 +nodelist: node-01 +output: /logs/service_dcgm-exporter_node-01.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-02 +nodelist: node-02 +output: /logs/service_dcgm-exporter_node-02.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-03 +nodelist: node-03 +output: /logs/service_dcgm-exporter_node-03.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_dcgm-exporter_node-04 +nodelist: node-04 +output: /logs/service_dcgm-exporter_node-04.out +container_image: nvcr.io#nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + dcgm-exporter --collect-interval=1000 --address :9401 + +## service_node-exporter_node-01 +nodelist: node-01 +output: /logs/service_node-exporter_node-01.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-02 +nodelist: node-02 +output: /logs/service_node-exporter_node-02.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-03 +nodelist: node-03 +output: /logs/service_node-exporter_node-03.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## service_node-exporter_node-04 +nodelist: node-04 +output: /logs/service_node-exporter_node-04.out +container_image: quay.io#prometheus/node-exporter:v1.8.2 +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +srun_export_env: + ETCD_ENDPOINTS: http://127.0.0.1:2379 +use_bash_wrapper: False +command: | + /bin/node_exporter --web.listen-address=:9101 --collector.disable-defaults --collector.cpu --collector.infiniband --collector.meminfo --collector.processes --collector.stat --collector.vmstat --collector.pressure --collector.meminfo_numa '--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pgsteal|pg.*fault).*' + +## tachometer +nodelist: node-01 +output: /logs/tachometer.out +srun_export_env: + POLARS_MAX_THREADS: 4 +use_bash_wrapper: False +command: | + tachometer-scraper --config /logs/tachometer_config.toml --local-dir /logs/tachometer/local --sync-interval 120 + +## +nodelist: node-01 +output: /logs/benchmark.out +container_image: llm-d-vllm +container_mounts: + /src/srtctl/benchmarks/scripts: /srtctl-benchmarks + /src/srtctl/runtime_scripts: /srtctl-runtime + : /model + /logs: /logs +env_to_set: + SRTCTL_FRONTEND_TYPE: llm-d + SRT_FRONTEND_HOST: 127.0.0.1 + SRT_FRONTEND_PORT: 8000 +command: | + bash /srtctl-benchmarks/sa-bench/bench.sh http://localhost:8000 128 128 16x32 inf /model Qwen/Qwen3-30B-A3B false 16 0 0 0.8 10 2 '' true random '' false diff --git a/tests/test_configs.py b/tests/test_configs.py index f4dd68dce..0eb9d610e 100644 --- a/tests/test_configs.py +++ b/tests/test_configs.py @@ -2599,8 +2599,8 @@ def test_dp_mode_detection(self): "decode": RoleConfig(args={"tensor-parallel-size": 4}), } ) - assert backend._is_dp_mode("prefill") is False - assert backend._is_dp_mode("decode") is False + assert backend.is_dp_mode("prefill") is False + assert backend.is_dp_mode("decode") is False # DP mode detected when data-parallel-size is set backend_dp = VLLMBackend( @@ -2609,8 +2609,8 @@ def test_dp_mode_detection(self): "decode": RoleConfig(args={"data-parallel-size": 16, "enable-expert-parallel": True}), } ) - assert backend_dp._is_dp_mode("prefill") is True - assert backend_dp._is_dp_mode("decode") is True + assert backend_dp.is_dp_mode("prefill") is True + assert backend_dp.is_dp_mode("decode") is True assert backend_dp._get_dp_size("prefill") == 16 def test_dp_per_gpu_mode_creates_per_gpu_processes(self): diff --git a/tests/test_e2e.py b/tests/test_e2e.py index ecc049143..75af058dc 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -30,7 +30,7 @@ def test_interactive_discovers_curated_examples(): assert SGLANG_ROUTER_DISAGG in examples assert MOCKER_EXAMPLE in examples - assert len(TOPOLOGY_EXAMPLES) == 22 + assert len(TOPOLOGY_EXAMPLES) == 25 # ============================================================================= diff --git a/tests/test_frontends.py b/tests/test_frontends.py index 074206f66..67b8ba686 100644 --- a/tests/test_frontends.py +++ b/tests/test_frontends.py @@ -69,6 +69,7 @@ def test_registry_lists_every_frontend_type(self): assert list_frontend_types() == [ "atomesh", "dynamo", + "llm-d", "none", "sglang", "sglang-router", @@ -222,7 +223,7 @@ def test_schema_rejects_unknown_type_at_load(self): from srtctl.core.schema import FrontendConfig, ResourceConfig, RoleConfig, SrtConfig with pytest.raises( - ValidationError, match="Unknown frontend.type 'toy-router'.*Available: atomesh, dynamo, none" + ValidationError, match="Unknown frontend.type 'toy-router'.*Available: atomesh, dynamo, llm-d, none" ): SrtConfig( name="toy", diff --git a/tests/test_launch_plan.py b/tests/test_launch_plan.py deleted file mode 100644 index 24989f137..000000000 --- a/tests/test_launch_plan.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for opt-in recording of realized Slurm launch commands.""" - -import json -import subprocess -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -from srtctl.core.launch_plan import configure_launch_plan -from srtctl.core.schema import ClusterConfig, OutputConfig -from srtctl.core.slurm import start_srun_process - - -@pytest.fixture(autouse=True) -def _reset_launch_plan(): - configure_launch_plan(None) - yield - configure_launch_plan(None) - - -def _launch(tmp_path: Path, **kwargs) -> Path: - plan_dir = tmp_path / "logs" / "launch-plan" - configure_launch_plan(plan_dir, job_id="42042") - with ( - patch("srtctl.core.slurm.get_slurm_job_id", return_value="42042"), - patch("srtctl.core.slurm._get_cluster_bash_preamble", return_value=None), - patch("subprocess.Popen", return_value=MagicMock()), - ): - start_srun_process(["python3", "-m", "worker"], **kwargs) - return plan_dir - - -def test_launch_plan_is_opt_in_for_recipe_and_cluster() -> None: - assert OutputConfig().record_launch_plan is False - assert OutputConfig(record_launch_plan=True).record_launch_plan is True - assert ClusterConfig().record_launch_plan is False - assert ClusterConfig(record_launch_plan=True).record_launch_plan is True - - -def test_disabled_recorder_writes_nothing(tmp_path: Path) -> None: - with ( - patch("srtctl.core.slurm.get_slurm_job_id", return_value="42042"), - patch("srtctl.core.slurm._get_cluster_bash_preamble", return_value=None), - patch("subprocess.Popen", return_value=MagicMock()), - ): - start_srun_process(["python3", "-m", "worker"]) - - assert not (tmp_path / "launch-plan").exists() - - -def test_records_exact_realized_srun_as_executable_script(tmp_path: Path) -> None: - output = tmp_path / "logs" / "node-a_prefill_w0.out" - plan_dir = _launch( - tmp_path, - nodelist=["node-a"], - output=str(output), - container_image="/containers/vllm.sqsh", - container_mounts={Path("/models/model"): Path("/model")}, - env_to_set={"NCCL_DEBUG": "INFO"}, - het_group=0, - ) - - scripts = list(plan_dir.glob("*.sh")) - assert [path.name for path in scripts] == ["001-node-a_prefill_w0.sh"] - script = scripts[0].read_text() - assert scripts[0].stat().st_mode & 0o100 - assert "exec srun --jobid 42042 --overlap" in script - assert "--nodelist node-a" in script - assert "--het-group=0" in script - assert "--container-image /containers/vllm.sqsh" in script - assert "export NCCL_DEBUG=INFO" in script - assert "python3 -m worker" in script - syntax = subprocess.run(["bash", "-n", str(scripts[0])], capture_output=True, text=True, check=False) - assert syntax.returncode == 0, syntax.stderr - - manifest = json.loads((plan_dir / "manifest.json").read_text()) - assert manifest["job_id"] == "42042" - assert manifest["steps"][0]["script"] == scripts[0].name - assert manifest["steps"][0]["nodelist"] == ["node-a"] - assert manifest["steps"][0]["container_image"] == "/containers/vllm.sqsh" - - -def test_secret_environment_values_are_not_persisted(tmp_path: Path) -> None: - plan_dir = _launch( - tmp_path, - output=str(tmp_path / "logs" / "benchmark.out"), - env_to_set={"HF_TOKEN": "hf-super-secret", "PUBLIC_VALUE": "visible"}, - srun_export_env={"AWS_SECRET_ACCESS_KEY": "aws-super-secret", "ENROOT_REMAP_ROOT": "yes"}, - ) - - script = next(plan_dir.glob("*.sh")).read_text() - manifest_text = (plan_dir / "manifest.json").read_text() - combined = script + manifest_text - assert "hf-super-secret" not in combined - assert "aws-super-secret" not in combined - assert 'export HF_TOKEN="${HF_TOKEN:?Set HF_TOKEN to replay this step}"' in script - assert "--export=ALL,AWS_SECRET_ACCESS_KEY,ENROOT_REMAP_ROOT=yes" in script - assert "export PUBLIC_VALUE=visible" in script - - manifest = json.loads(manifest_text) - assert manifest["steps"][0]["required_secret_environment"] == ["AWS_SECRET_ACCESS_KEY", "HF_TOKEN"] - - -def test_multiple_commands_have_stable_order_and_distinct_files(tmp_path: Path) -> None: - plan_dir = _launch(tmp_path, output=str(tmp_path / "logs" / "infra.out")) - with ( - patch("srtctl.core.slurm.get_slurm_job_id", return_value="42042"), - patch("srtctl.core.slurm._get_cluster_bash_preamble", return_value=None), - patch("subprocess.Popen", return_value=MagicMock()), - ): - start_srun_process(["python3", "bench.py"], output=str(tmp_path / "logs" / "benchmark.out")) - - manifest = json.loads((plan_dir / "manifest.json").read_text()) - assert [entry["sequence"] for entry in manifest["steps"]] == [1, 2] - assert [entry["script"] for entry in manifest["steps"]] == ["001-infra.sh", "002-benchmark.sh"] diff --git a/tests/test_llm_d_frontend.py b/tests/test_llm_d_frontend.py new file mode 100644 index 000000000..aede3fa31 --- /dev/null +++ b/tests/test_llm_d_frontend.py @@ -0,0 +1,662 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 SemiAnalysis LLC. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""llm-d router frontend (`frontend.type: llm-d`) and its P/D sidecar service.""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +import yaml +from marshmallow import ValidationError + +from srtctl.backends import VLLMBackend +from srtctl.core.config import load_config +from srtctl.core.schema import SrtConfig +from srtctl.core.topology import Endpoint, Process +from srtctl.frontends import LLMDFrontend, get_frontend, list_frontend_types +from srtctl.frontends.llm_d import ( + DISCOVERY_PLUGIN, + ENDPOINTS_FILE, + ENVOY_CONFIG_FILE, + EPP_CONFIG_FILE, + epp_config_document, + parse_ready_endpoints, + with_kv_events, +) +from srtctl.ports import ( + LLM_D_ENVOY_ADMIN_PORT, + LLM_D_EPP_GRPC_PORT, + LLM_D_EPP_KV_EVENTS_PORT, + LLM_D_EPP_METRICS_PORT, + WORKER_PROXY_PORT_BASE, +) +from srtctl.services.implicit import effective_services +from srtctl.services.llm_d_sidecar import sidecar_kv_connector +from srtctl.services.registry import ServiceLaunchContext, get_service_kind +from tests.launch_snapshots import EXAMPLES_DIR, render_launch_plan + +DISAGG = EXAMPLES_DIR / "vllm/llm-d-disagg.yaml" +AGG = EXAMPLES_DIR / "vllm/llm-d-agg.yaml" + + +def _recipe(path: Path = DISAGG) -> dict: + return yaml.safe_load(path.read_text()) + + +def _load(recipe: dict) -> SrtConfig: + return SrtConfig.Schema().load(recipe) + + +def test_registry_resolves_llm_d() -> None: + frontend = get_frontend("llm-d") + assert isinstance(frontend, LLMDFrontend) + assert frontend.required_backend == "vllm" + assert frontend.worker_api_port("decode") == "allocated" + assert frontend.frontend_metrics_port(None) == LLM_D_EPP_METRICS_PORT + + +def test_pd_proxies_decode_workers_and_implies_the_sidecar() -> None: + config = load_config(DISAGG) + assert LLMDFrontend().proxied_worker_modes(config) == frozenset({"decode"}) + sidecar = next(entry for entry in effective_services(config) if entry.service.type == "llm-d-sidecar") + assert sidecar.implicit + assert sidecar.service.effective_placement == "decode" + assert sidecar.service.effective_per == "worker" + assert sidecar.service.effective_start == "before_workers" + assert sidecar.service.effective_critical + + +def test_aggregate_job_has_no_proxy_and_no_sidecar() -> None: + config = load_config(AGG) + assert LLMDFrontend().proxied_worker_modes(config) == frozenset() + assert not any(entry.service.type == "llm-d-sidecar" for entry in effective_services(config)) + + +def test_only_routable_decode_workers_get_a_proxy_port() -> None: + """A multi-node decode worker has one API (its leader); the follower gets neither port.""" + recipe = _recipe() + recipe["roles"]["decode"] = {**recipe["roles"]["decode"], "nodes": 2, "gpus": 16} + config = _load(recipe) + endpoints = [ + Endpoint("prefill", 0, ("node0",), frozenset({0})), + Endpoint("decode", 0, ("node1", "node2"), frozenset(range(8))), + ] + processes = config.worker_processes(endpoints) + by_role = {(p.endpoint_mode, p.node_rank): p for p in processes} + assert by_role[("prefill", 0)].proxy_port is None + assert by_role[("decode", 0)].proxy_port == WORKER_PROXY_PORT_BASE + assert by_role[("decode", 1)].http_port == 0 + assert by_role[("decode", 1)].proxy_port is None + kind = get_service_kind("llm-d-sidecar") + assert [kind.attaches_to(p) for p in processes] == [False, True, False] + + +def test_sidecar_command_binds_the_proxy_port_in_front_of_the_worker() -> None: + config = load_config(DISAGG) + process = Process("node1", frozenset({0}), 7500, 6100, "decode", 0, proxy_port=9600) + service = next(e.service for e in effective_services(config) if e.service.type == "llm-d-sidecar") + ctx = ServiceLaunchContext( + runtime=MagicMock(), + node="node1", + node_ip="10.0.0.2", + node_id=0, + index=0, + role="decode", + process=process, + config=config, + ) + kind = get_service_kind("llm-d-sidecar") + assert kind.build_command(service, ctx) == [ + "pd-sidecar", + "--port=9600", + "--model-server-port=6100", + "--kv-connector=nixlv2", + "--secure-proxy=false", + ] + probe = kind.readiness(service, ctx) + assert probe is not None and probe.http is not None + assert (probe.http.port, probe.http.path) == (9600, "/health") + assert ctx.template_vars()["worker_proxy_port"] == "9600" + assert ctx.template_vars()["worker_http_port"] == "6100" + + +def test_declared_sidecar_takes_over_with_its_own_command_and_args() -> None: + recipe = _recipe() + recipe["services"] = [ + { + "name": "llm-d-sidecar", + "type": "llm-d-sidecar", + "command": ["/app/pd-sidecar"], + "args": ["--enable-prefiller-sampling"], + } + ] + config = _load(recipe) + service = next(e.service for e in effective_services(config) if e.service.type == "llm-d-sidecar") + command = get_service_kind("llm-d-sidecar").build_command(service, ServiceLaunchContext.preview()) + assert command[0] == "/app/pd-sidecar" + assert command[-1] == "--enable-prefiller-sampling" + + +def test_declared_sidecar_must_sit_on_proxied_workers() -> None: + recipe = _recipe(AGG) + recipe["services"] = [{"name": "llm-d-sidecar", "type": "llm-d-sidecar"}] + with pytest.raises(ValidationError, match="fronts the workers the frontend proxies"): + _load(recipe) + + +@pytest.mark.parametrize( + "service", + [ + {"name": "llm-d-sidecar", "type": "llm-d-sidecar", "enabled": False}, + {"name": "llm-d-sidecar", "type": "generic", "command": ["sleep", "infinity"]}, + {"name": "extra-sidecar", "type": "llm-d-sidecar"}, + ], +) +def test_pd_requires_exactly_one_enabled_sidecar_service(service: dict) -> None: + recipe = _recipe() + recipe["services"] = [service] + with pytest.raises(ValidationError, match="requires exactly one enabled llm-d-sidecar"): + _load(recipe) + + +def test_sidecar_can_be_renamed_when_the_implied_one_is_disabled() -> None: + recipe = _recipe() + recipe["services"] = [ + {"name": "llm-d-sidecar", "type": "llm-d-sidecar", "enabled": False}, + {"name": "custom-sidecar", "type": "llm-d-sidecar"}, + ] + sidecars = [entry.service for entry in effective_services(_load(recipe)) if entry.service.type == "llm-d-sidecar"] + assert [service.name for service in sidecars] == ["custom-sidecar"] + + +@pytest.mark.parametrize("field", ["command", "args"]) +@pytest.mark.parametrize("flag", ["port", "model-server-port", "kv-connector", "secure-proxy"]) +@pytest.mark.parametrize("equals", [False, True]) +def test_sidecar_rejects_managed_flags_at_config_load(field: str, flag: str, equals: bool) -> None: + recipe = _recipe() + argv = [f"--{flag}=override"] if equals else [f"--{flag}", "override"] + recipe["services"] = [ + {"name": "llm-d-sidecar", "type": "llm-d-sidecar", field: ["pd-sidecar", *argv] if field == "command" else argv} + ] + with pytest.raises(ValidationError, match="which srtctl manages"): + _load(recipe) + + +def test_pd_epp_config_needs_scheduling_profiles() -> None: + recipe = _recipe() + del recipe["frontend"]["epp_config"]["schedulingProfiles"] + with pytest.raises(ValidationError, match="needs frontend.epp_config"): + _load(recipe) + + +@pytest.mark.parametrize("frontend_type", [name for name in list_frontend_types() if name != "llm-d"] + ["none"]) +@pytest.mark.parametrize("epp_config", [{}, {"plugins": [{"type": "queue-scorer"}]}]) +def test_epp_config_needs_the_llm_d_frontend(frontend_type: str, epp_config: dict) -> None: + recipe = _recipe(AGG) + recipe["frontend"].update(type=frontend_type, epp_config=epp_config) + if frontend_type == "none": + recipe.pop("roles") + recipe.pop("engine") + recipe["services"] = [{"name": "sleeper", "command": ["sleep", "infinity"], "nodes": 1}] + with pytest.raises( + ValidationError, match=f"frontend.epp_config is not supported with frontend.type: {frontend_type}" + ): + _load(recipe) + + +@pytest.mark.parametrize("path", [AGG, DISAGG]) +def test_without_epp_config_srtctl_runs_the_guide_scorers(path: Path, tmp_path: Path) -> None: + """The EPP's own default only applies to a plugin-less file; srtctl's always carries discovery.""" + recipe = _recipe(path) + del recipe["frontend"]["epp_config"] + config = _load(recipe) + runtime = SimpleNamespace( + network_interface=None, + log_dir=tmp_path, + container_image="model.sqsh", + container_mounts={}, + environment={}, + srun_options={}, + nodes=SimpleNamespace(het_group_for=lambda node: None), + ) + processes = [Process("node0", frozenset({0}), 7500, 6100, "agg", 0)] + with ( + patch("srtctl.frontends.static_router.get_hostname_ip", return_value="10.0.0.1"), + patch.object(LLMDFrontend, "wait_for_workers"), + patch.object(LLMDFrontend, "start_process", return_value=MagicMock()), + ): + LLMDFrontend().start_frontends( + SimpleNamespace(frontend_nodes=["node0"], frontend_port=8000), runtime, config, config.backend, processes + ) + epp = yaml.safe_load((tmp_path / EPP_CONFIG_FILE).read_text()) + profiles = {profile["name"]: profile["plugins"] for profile in epp["schedulingProfiles"]} + guide = [ + {"pluginRef": "queue-scorer", "weight": 2}, + {"pluginRef": "kv-cache-utilization-scorer", "weight": 2}, + {"pluginRef": "prefix-cache-scorer", "weight": 3}, + {"pluginRef": "no-hit-lru-scorer", "weight": 2}, + ] + if path == AGG: + assert profiles == {"default": guide} + else: + assert profiles == { + "prefill": [{"pluginRef": "prefill-filter"}, *guide], + "decode": [{"pluginRef": "decode-filter"}, *guide[:2]], + } + assert {"type": "always-disagg-pd-decider"} in epp["plugins"] + assert epp["plugins"][-1]["type"] == "file-discovery" + + +def test_recipe_cannot_configure_discovery() -> None: + recipe = _recipe() + recipe["frontend"]["epp_config"]["dataLayer"] = {"discovery": {"pluginRef": "mine"}} + with pytest.raises(ValidationError, match="must not configure endpoint discovery"): + _load(recipe) + + +def test_recipe_cannot_reuse_the_managed_discovery_plugin_name() -> None: + recipe = _recipe(AGG) + recipe["frontend"]["epp_config"]["plugins"].append({"type": "queue-scorer", "name": DISCOVERY_PLUGIN}) + with pytest.raises(ValidationError, match="must not configure endpoint discovery"): + _load(recipe) + + +def test_tokenizer_url_is_managed_even_without_precise_routing() -> None: + recipe = _recipe(AGG) + recipe["frontend"]["epp_config"]["plugins"].append( + {"type": "token-producer", "parameters": {"vllm": {"url": "http://custom-tokenizer:8000"}}} + ) + with pytest.raises(ValidationError, match="sets token-producer vllm.url"): + _load(recipe) + + +def test_decode_connector_needs_a_sidecar_protocol() -> None: + recipe = _recipe() + recipe["engine"]["connector"] = "lmcache" + with pytest.raises(ValidationError, match="no protocol for the vllm decode KV connector 'LMCacheConnectorV1'"): + _load(recipe) + + +def test_explicit_kv_transfer_config_selects_the_protocol() -> None: + """A role's own --kv-transfer-config wins over engine.connector, as on the command line.""" + recipe = _recipe() + recipe["engine"]["connector"] = None + recipe["roles"]["decode"]["args"]["kv-transfer-config"] = '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + config = _load(recipe) + assert isinstance(config.backend, VLLMBackend) + assert config.backend.kv_connector_classes("decode") == ("NixlConnector",) + assert config.backend.kv_connector_classes("prefill") == () + + +def test_multi_connector_maps_through_its_transfer_connector() -> None: + """NIXL next to CPU offloading (llm-d's tiered wide-EP guide) still speaks nixlv2.""" + recipe = _recipe() + recipe["engine"]["connector"] = None + recipe["roles"]["decode"]["args"]["kv-transfer-config"] = { + "kv_connector": "MultiConnector", + "kv_role": "kv_both", + "kv_connector_extra_config": { + "connectors": [ + {"kv_connector": "OffloadingConnector", "kv_role": "kv_both"}, + {"kv_connector": "NixlConnector", "kv_role": "kv_both"}, + ] + }, + } + config = _load(recipe) + assert config.backend.kv_connector_classes("decode") == ("MultiConnector", "OffloadingConnector", "NixlConnector") + assert sidecar_kv_connector(config.backend) == "nixlv2" + + +@pytest.mark.parametrize("additional_frontends", [0, 1, 9]) +def test_multinode_nginx_is_rejected_even_with_one_router(additional_frontends: int) -> None: + recipe = _recipe() + recipe["frontend"]["enable_multiple_frontends"] = True + recipe["frontend"]["num_additional_frontends"] = additional_frontends + recipe["roles"]["decode"]["nodes"] = 1 + with pytest.raises(ValidationError, match="Set frontend.enable_multiple_frontends: false"): + _load(recipe) + + +def test_single_node_allows_multiple_frontends_setting_without_nginx() -> None: + recipe = _recipe(AGG) + recipe["frontend"]["enable_multiple_frontends"] = True + config = _load(recipe) + assert config.engine_node_count == 1 + + +def test_epp_config_gets_srtctl_discovery() -> None: + user = {"plugins": [{"type": "queue-scorer"}], "dataLayer": {"sources": [{"pluginRef": "m"}]}} + original = copy.deepcopy(user) + document = epp_config_document(user, "/logs/endpoints.yaml") + assert user == original + assert list(document)[:2] == ["apiVersion", "kind"] + assert document["plugins"][-1] == { + "name": DISCOVERY_PLUGIN, + "type": "file-discovery", + "parameters": {"path": "/logs/endpoints.yaml", "watchFile": False}, + } + assert document["dataLayer"] == {"sources": [{"pluginRef": "m"}], "discovery": {"pluginRef": DISCOVERY_PLUGIN}} + + +def test_start_frontends_writes_the_router_files_and_starts_epp_then_envoy(tmp_path: Path) -> None: + config = load_config(DISAGG) + runtime = SimpleNamespace( + network_interface=None, + log_dir=tmp_path, + container_image="model.sqsh", + container_mounts={}, + environment={}, + srun_options={}, + nodes=SimpleNamespace(het_group_for=lambda node: None), + ) + processes = [ + Process("node0", frozenset({0}), 7500, 6100, "prefill", 0), + Process("node1", frozenset({0}), 7501, 6100, "decode", 0, proxy_port=9600), + ] + ips = {"node0": "10.0.0.1", "node1": "10.0.0.2"} + with ( + patch("srtctl.frontends.static_router.get_hostname_ip", side_effect=lambda node, _: ips[node]), + patch.object(LLMDFrontend, "wait_for_workers") as wait, + patch.object(LLMDFrontend, "start_process", return_value=MagicMock()) as start, + ): + managed = LLMDFrontend().start_frontends( + SimpleNamespace(frontend_nodes=["node0"], frontend_port=8000), runtime, config, config.backend, processes + ) + + # The gate probes the workers themselves (vLLM /health), not the sidecar, which answers at once. + assert [w.url for w in wait.call_args.args[0]] == ["http://10.0.0.1:6100", "http://10.0.0.2:6100"] + endpoints = yaml.safe_load((tmp_path / ENDPOINTS_FILE).read_text())["endpoints"] + assert [(e["address"], e["port"], e["labels"]["llm-d.ai/role"]) for e in endpoints] == [ + ("10.0.0.1", "6100", "prefill"), + ("10.0.0.2", "9600", "decode"), + ] + epp = yaml.safe_load((tmp_path / EPP_CONFIG_FILE).read_text()) + assert epp["dataLayer"]["discovery"] == {"pluginRef": DISCOVERY_PLUGIN} + assert {"name": "prefill", "plugins": epp["schedulingProfiles"][0]["plugins"]} == epp["schedulingProfiles"][0] + envoy = yaml.safe_load((tmp_path / ENVOY_CONFIG_FILE).read_text()) + assert envoy["admin"]["address"]["socket_address"]["port_value"] == LLM_D_ENVOY_ADMIN_PORT + listener = envoy["static_resources"]["listeners"][0]["address"]["socket_address"] + assert listener == {"address": "0.0.0.0", "port_value": 8000} + routes = envoy["static_resources"]["listeners"][0]["filter_chains"][0]["filters"][0]["typed_config"]["route_config"] + metrics_route = routes["virtual_hosts"][0]["routes"][0] + assert metrics_route["match"] == {"path": "/metrics"} and metrics_route["direct_response"] == {"status": 404} + epp_cluster = envoy["static_resources"]["clusters"][1]["load_assignment"]["endpoints"][0]["lb_endpoints"][0] + assert epp_cluster["endpoint"]["address"]["socket_address"]["port_value"] == LLM_D_EPP_GRPC_PORT + + assert [p.name for p in managed] == ["llm-d-epp_0", "llm-d-envoy_0"] + assert all(p.critical for p in managed) + epp_cmd, envoy_cmd = (call.kwargs["command"] for call in start.call_args_list) + assert epp_cmd[0] == "epp" and "--config-file=/logs/llm-d-epp-config.yaml" in epp_cmd + assert "--secure-serving=false" in epp_cmd + assert envoy_cmd == ["envoy", "-c", "/logs/llm-d-envoy.yaml", "--disable-hot-restart"] + + +@pytest.mark.parametrize( + "flag", + [ + "pool-name", + "pool-namespace", + "config-file", + "config-text", + "grpc-port", + "grpc-health-port", + "metrics-port", + "secure-serving", + ], +) +@pytest.mark.parametrize("underscores", [False, True]) +def test_managed_epp_flags_are_rejected_at_config_load(flag: str, underscores: bool) -> None: + recipe = _recipe(AGG) + recipe["frontend"]["args"] = {flag.replace("-", "_") if underscores else flag: "override"} + with pytest.raises(ValidationError, match=f"{flag}, which srtctl manages"): + _load(recipe) + + +def test_ready_endpoints_gauge_is_parsed() -> None: + text = ( + "# HELP llm_d_epp_ready_endpoints The number of ready endpoints.\n" + "# TYPE llm_d_epp_ready_endpoints gauge\n" + 'llm_d_epp_ready_endpoints{name="srtctl"} 2\n' + "llm_d_epp_ready_endpoints_total 9\n" + ) + assert parse_ready_endpoints(text) == 2 + assert parse_ready_endpoints("# nothing yet\n") is None + + +@pytest.mark.parametrize(("ready", "expected"), [(1, False), (2, True)]) +def test_probe_needs_envoy_ready_and_every_endpoint_scraped(ready: int, expected: bool) -> None: + def get(url: str, timeout: float) -> SimpleNamespace: + if url.endswith(f":{LLM_D_ENVOY_ADMIN_PORT}/ready"): + return SimpleNamespace(status_code=200, text="LIVE") + assert url.endswith(f":{LLM_D_EPP_METRICS_PORT}/metrics") + return SimpleNamespace(status_code=200, text=f'llm_d_epp_ready_endpoints{{name="srtctl"}} {ready}\n') + + with patch("srtctl.frontends.llm_d.requests.get", side_effect=get): + result = LLMDFrontend().probe_ready("node0", 8000, 1, 1, None) + assert result.ready is expected + assert f"{ready}/2" in result.message + + +def test_probe_waits_for_envoy() -> None: + with patch("srtctl.frontends.llm_d.requests.get", return_value=SimpleNamespace(status_code=503, text="")): + assert not LLMDFrontend().probe_ready("node0", 8000, 0, 2, None).ready + + +def test_example_launches_sidecar_workers_epp_and_envoy() -> None: + """Through the mock orchestrator: the sidecar starts before the workers, the router after them.""" + plan = render_launch_plan(DISAGG) + assert "# exit_code: 0" in plan + steps = [line[3:] for line in plan.splitlines() if line.startswith("## ")] + assert steps[:5] == [ + "service_llm-d-sidecar_decode_0_node-02", + "prefill_0_node-01", + "decode_0_node-02", + "llm-d-epp_0", + "llm-d-envoy_0", + ] + sidecar = next(line.strip() for line in plan.splitlines() if line.strip().startswith("pd-sidecar")) + decode = plan.split("## decode_0_node-02")[1].split("## ")[0] + assert f"--port={WORKER_PROXY_PORT_BASE}" in sidecar + assert "--model-server-port=6100" in sidecar and "--port 6100" in decode + assert "--kv-connector=nixlv2" in sidecar + + +def test_agg_example_launches_epp_and_envoy() -> None: + plan = render_launch_plan(AGG) + assert "# exit_code: 0" in plan + assert "## llm-d-epp_0" in plan and "## llm-d-envoy_0" in plan + assert "pd-sidecar" not in plan + + +def test_engine_must_be_vllm() -> None: + recipe = _recipe(AGG) + recipe["engine"] = "sglang" + recipe["roles"]["agg"]["args"] = {"served-model-name": "m"} + with pytest.raises(ValidationError, match="requires backend"): + _load(recipe) + + +DP_RANKS = EXAMPLES_DIR / "vllm/llm-d-dp-ranks.yaml" + + +def _precise(recipe: dict) -> dict: + """The disaggregated example with precise prefix-cache routing on its prefill worker.""" + epp = recipe["frontend"]["epp_config"] + epp["plugins"] = [ + {"type": "token-producer"}, + {"type": "precise-prefix-cache-producer"}, + *(plugin for plugin in epp["plugins"] if plugin["type"] != "approx-prefix-cache-producer"), + ] + recipe["roles"]["prefill"]["kv_events"] = True + return recipe + + +def test_external_lb_ranks_are_endpoints_with_their_own_ports() -> None: + """data-parallel-external-lb: one `vllm serve` per DP rank, each an endpoint on its own port.""" + config = load_config(DP_RANKS) + endpoints = [Endpoint("agg", 0, ("node0", "node1"), frozenset(range(8)))] + processes = config.worker_processes(endpoints) + assert [p.dp_rank for p in processes] == list(range(16)) + assert all(p.http_port > 0 for p in processes) + assert len({(p.node, p.http_port) for p in processes}) == 16 + assert processes[9].engine_suffix == "_dp9" + with patch("srtctl.frontends.static_router.get_hostname_ip", side_effect=lambda node, _: node): + document = LLMDFrontend().endpoints_document(processes, None) + assert len(document["endpoints"]) == 16 + frontend = LLMDFrontend() + assert frontend.health_expectations(config, processes) == (0, 16, "16 llm-d endpoints") + assert frontend.worker_metrics_port(processes[9], MagicMock()) == processes[9].http_port + + +def test_external_lb_ranks_publish_kv_events_to_the_epp() -> None: + """Every rank connects to the EPP's one socket (vLLM adds the rank back) under its own endpoint's topic.""" + plan = render_launch_plan(DP_RANKS) + assert "# exit_code: 0" in plan + steps = [line[3:] for line in plan.splitlines() if line.startswith("## ")] + assert len(steps) == len(set(steps)) + assert "agg_0_node-02_dp9" in steps + rank9 = plan.split("## agg_0_node-02_dp9")[1].split("## ")[0] + assert "--data-parallel-rank 9 --data-parallel-address" in rank9 + assert "--data-parallel-external-lb" in rank9 and "--headless" not in rank9 + kv_events = json.loads(rank9.split("--kv-events-config '")[1].split("'")[0]) + port = rank9.split("--port ")[1].split()[0] + assert kv_events["endpoint"] == f"tcp://127.0.0.1:{LLM_D_EPP_KV_EVENTS_PORT - 9}" + assert kv_events["topic"] == f"kv@127.0.0.1:{port}@Qwen/Qwen3-30B-A3B" + assert kv_events["publisher"] == "zmq" and kv_events["enable_kv_cache_events"] is True + + +def test_external_lb_decode_ranks_each_get_a_sidecar() -> None: + recipe = _precise(_recipe()) + for role in ("prefill", "decode"): + recipe["roles"][role]["gpus"] = 2 + recipe["roles"][role]["args"].update({"data-parallel-size": 2, "data-parallel-external-lb": True}) + config = _load(recipe) + endpoints = [ + Endpoint("prefill", 0, ("node0",), frozenset({0, 1})), + Endpoint("decode", 0, ("node1",), frozenset({0, 1})), + ] + processes = config.worker_processes(endpoints) + decode = [p for p in processes if p.endpoint_mode == "decode"] + assert [p.proxy_port for p in decode] == [WORKER_PROXY_PORT_BASE, WORKER_PROXY_PORT_BASE + 1] + kind = get_service_kind("llm-d-sidecar") + assert [kind.attaches_to(p) for p in processes] == [False, False, True, True] + # The decode topic names the sidecar, the endpoint the EPP routes to. + runtime = SimpleNamespace(network_interface=None, head_node_ip="10.0.0.9") + with patch("srtctl.frontends.static_router.get_hostname_ip", return_value="10.0.0.2"): + assert LLMDFrontend().kv_events_subscriber(decode[1], runtime, "m") == ( + "10.0.0.9", + LLM_D_EPP_KV_EVENTS_PORT, + f"kv@10.0.0.2:{WORKER_PROXY_PORT_BASE + 1}@m", + ) + + +def test_kv_events_need_the_precise_producer_and_vice_versa() -> None: + recipe = _recipe() + recipe["roles"]["prefill"]["kv_events"] = True + with pytest.raises(ValidationError, match="add a precise-prefix-cache-producer plugin"): + _load(recipe) + recipe = _precise(_recipe()) + del recipe["roles"]["prefill"]["kv_events"] + with pytest.raises(ValidationError, match="set roles..kv_events"): + _load(recipe) + + +@pytest.mark.parametrize("field", ["endpoint", "topic"]) +def test_worker_kv_event_routing_cannot_be_overridden(field: str) -> None: + recipe = _precise(_recipe()) + recipe["roles"]["prefill"]["kv_events"] = {field: "override"} + with pytest.raises(ValidationError, match=f"roles.prefill.kv_events sets {field}, which srtctl manages"): + _load(recipe) + + +@pytest.mark.parametrize( + ("edit", "message"), + [ + ( + lambda r: r["frontend"]["epp_config"]["plugins"][1].update( + {"parameters": {"kvEventsConfig": {"zmqEndpoint": "tcp://*:1"}}} + ), + "kvEventsConfig.zmqEndpoint, which srtctl manages", + ), + (lambda r: r["frontend"]["epp_config"]["plugins"].pop(0), "add a token-producer plugin"), + ( + lambda r: r["frontend"]["epp_config"]["plugins"][0].update({"parameters": {"vllm": {"url": "http://x"}}}), + "sets token-producer vllm.url", + ), + ( + lambda r: r["frontend"].update({"placement": {"node": "first_decode"}}), + "publish to the EPP on the head node", + ), + ( + lambda r: r["roles"]["prefill"].update( + {"gpus": 2, "args": {**r["roles"]["prefill"]["args"], "data-parallel-size": 2}} + ), + "set roles.prefill.args.data-parallel-external-lb: true", + ), + ], +) +def test_precise_prefix_routing_rules(edit, message: str) -> None: + recipe = _precise(_recipe()) + edit(recipe) + with pytest.raises(ValidationError, match=message): + _load(recipe) + + +def test_with_kv_events_wires_the_socket_and_the_render_url() -> None: + document = { + "plugins": [ + {"type": "token-producer"}, + {"type": "token-producer", "name": "approx", "parameters": {"estimate": {}}}, + {"type": "precise-prefix-cache-producer", "parameters": {"kvEventsConfig": {"concurrency": 8}}}, + {"type": "queue-scorer"}, + ] + } + with_kv_events(document, "http://10.0.0.1:6100", "m") + token, estimate, producer, scorer = document["plugins"] + assert token["parameters"] == {"modelName": "m", "vllm": {"url": "http://10.0.0.1:6100"}} + assert estimate["parameters"] == {"estimate": {}} + assert producer["parameters"]["kvEventsConfig"] == { + "concurrency": 8, + "zmqEndpoint": f"tcp://*:{LLM_D_EPP_KV_EVENTS_PORT}", + "discoverPods": False, + } + assert scorer == {"type": "queue-scorer"} + + +def test_start_frontends_points_the_token_producer_at_the_prefill_worker(tmp_path: Path) -> None: + config = _load(_precise(_recipe())) + runtime = SimpleNamespace( + network_interface=None, + log_dir=tmp_path, + container_image="model.sqsh", + container_mounts={}, + environment={}, + srun_options={}, + model_path=Path("/models/qwen"), + nodes=SimpleNamespace(het_group_for=lambda node: None), + ) + processes = [ + Process("node1", frozenset({0}), 7501, 6100, "decode", 0, proxy_port=9600), + Process("node0", frozenset({0}), 7500, 6101, "prefill", 0), + ] + ips = {"node0": "10.0.0.1", "node1": "10.0.0.2"} + with ( + patch("srtctl.frontends.static_router.get_hostname_ip", side_effect=lambda node, _: ips[node]), + patch.object(LLMDFrontend, "wait_for_workers"), + patch.object(LLMDFrontend, "start_process", return_value=MagicMock()), + ): + LLMDFrontend().start_frontends( + SimpleNamespace(frontend_nodes=["node0"], frontend_port=8000), runtime, config, config.backend, processes + ) + plugins = {p["type"]: p for p in yaml.safe_load((tmp_path / EPP_CONFIG_FILE).read_text())["plugins"]} + assert plugins["token-producer"]["parameters"] == { + "modelName": "Qwen/Qwen3-0.6B", + "vllm": {"url": "http://10.0.0.1:6101"}, + } + assert plugins["precise-prefix-cache-producer"]["parameters"]["kvEventsConfig"]["discoverPods"] is False diff --git a/tests/test_port_allocator.py b/tests/test_port_allocator.py index e34f80130..714fd7892 100644 --- a/tests/test_port_allocator.py +++ b/tests/test_port_allocator.py @@ -24,6 +24,7 @@ MORIIO_NOTIFY_PORTS, NCCL_PORTS, NIXL_PORTS, + PROXY_PORTS, SIDECAR_GRPC_PORTS, SYS_PORTS, TOKENSPEED_PORTS, @@ -96,6 +97,7 @@ def test_every_kind_has_its_own_range(self): TRTLLM_DIST_INIT_PORTS, TOKENSPEED_PORTS, SIDECAR_GRPC_PORTS, + PROXY_PORTS, ] assert len({kind.name for kind in kinds}) == len(kinds) assert len({kind.base for kind in kinds}) == len(kinds) @@ -134,6 +136,7 @@ def test_example_topologies_bind_no_port_twice(recipe: Path): "kv_events": process.kv_events_port, "kvbm_zmq": process.kvbm_zmq_port, "sidecar_grpc": process.sidecar_grpc_port, + "proxy": process.proxy_port, "nccl": process.nccl_port, "vllm_scan": process.vllm_scan_port, "moriio_handshake": process.moriio_handshake_port, diff --git a/tests/test_services.py b/tests/test_services.py index 672b112e4..a3fe6a856 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -112,6 +112,7 @@ def test_registered_kinds() -> None: "etcd", "generic", "gms", + "llm-d-sidecar", "lmcache-server", "mooncake-master", "mooncake-store", diff --git a/tests/test_slurm.py b/tests/test_slurm.py index 2ef1519df..2bf440437 100644 --- a/tests/test_slurm.py +++ b/tests/test_slurm.py @@ -231,6 +231,7 @@ def test_worker_stage_wraps_nonfatal_fingerprint_hook(tmp_path: Path) -> None: gpu_indices=list(range(8)), cuda_visible_devices="0,1,2,3,4,5,6,7", het_group=None, + dp_rank=None, trtllm_dist_init_port=29500, sidecar_grpc_port=50051, ) @@ -303,6 +304,7 @@ def _remap_worker_mixin(tmp_path: Path, *, frontend_type: str, dynamo_install: b gpu_indices=list(range(8)), cuda_visible_devices="0,1,2,3,4,5,6,7", het_group=None, + dp_rank=None, trtllm_dist_init_port=29500, sidecar_grpc_port=50051, ) @@ -883,6 +885,7 @@ def test_worker_stage_unsets_vllm_port_for_multinode_endpoint(tmp_path: Path) -> gpu_indices=list(range(8)), cuda_visible_devices="0,1,2,3,4,5,6,7", het_group=None, + dp_rank=None, trtllm_dist_init_port=29500, sidecar_grpc_port=50051, ) diff --git a/tests/test_vllm_connectors.py b/tests/test_vllm_connectors.py index 350664bfa..3a195c5a9 100644 --- a/tests/test_vllm_connectors.py +++ b/tests/test_vllm_connectors.py @@ -91,8 +91,8 @@ def test_direct_aggregate_worker_runs_only_its_role_connector(aggregated, expect from pathlib import Path from unittest.mock import MagicMock - from srtctl.core.topology import Process from srtctl.core.schema import RoleConfig + from srtctl.core.topology import Process backend = VLLMBackend(connector="nixl", roles={"agg": RoleConfig(args=aggregated)}) process = Process( @@ -113,3 +113,73 @@ def test_direct_aggregate_worker_runs_only_its_role_connector(aggregated, expect assert "--connector" not in cmd kv_config = json.loads(cmd[cmd.index("--kv-transfer-config") + 1]) if "--kv-transfer-config" in cmd else None assert kv_config == expected + + +@pytest.mark.parametrize("frontend,sidecar", [("llm-d", False), ("dynamo", False), ("dynamo", True)]) +@pytest.mark.parametrize("mode", ["agg", "decode"]) +@pytest.mark.parametrize("key", ["kv-transfer-config", "kv_transfer_config"]) +@pytest.mark.parametrize("as_mapping", [False, True]) +def test_explicit_transfer_config_is_shared_by_commands_and_sidecar(frontend, sidecar, mode, key, as_mapping): + """An explicit MultiConnector beats both aliases and reaches each engine exactly once.""" + import shlex + from pathlib import Path + from unittest.mock import MagicMock, patch + + from srtctl.core.schema import DynamoConfig + from srtctl.core.topology import Endpoint + from srtctl.services.llm_d_sidecar import sidecar_kv_connector + + payload = { + "kv_connector": "MultiConnector", + "kv_role": "kv_both", + "kv_connector_extra_config": { + "connectors": [{"kv_connector": "OffloadingConnector"}, {"kv_connector": "NixlConnector"}] + }, + } + explicit = payload if as_mapping else json.dumps(payload) + backend = VLLMBackend( + connector="lmcache-mp", + roles={mode: RoleConfig(args={"connector": "lmcache", key: explicit})}, + ) + assert json.loads(backend.kv_transfer_config(mode)) == payload + assert backend.kv_connector_classes(mode) == ("MultiConnector", "OffloadingConnector", "NixlConnector") + assert backend.kv_connector_for_mode(mode) is _CONNECTOR_MAP["lmcache"] + if mode == "decode": + assert sidecar_kv_connector(backend) == "nixlv2" + endpoint = Endpoint(mode, 0, ("node0",), frozenset({0})) + processes = backend.endpoints_to_processes([endpoint], frontend_type=frontend, dynamo_sidecar=sidecar) + assert all(p.moriio_handshake_port is None for p in processes) + runtime = MagicMock(model_path=Path("/model"), is_hf_model=False, frontend_port=9000) + runtime.dynamo = DynamoConfig(sidecar=sidecar) + with patch("srtctl.core.slurm.get_hostname_ip", return_value="127.0.0.1"): + command = backend.build_worker_command( + process=processes[0], endpoint_processes=processes, runtime=runtime, frontend_type=frontend + ) + if sidecar: + command = shlex.split( + next(line.removesuffix(" &") for line in command[2].splitlines() if line.startswith("vllm-rs serve ")) + ) + assert command.count("--kv-transfer-config") == 1 + assert json.loads(command[command.index("--kv-transfer-config") + 1]) == payload + assert "--connector" not in command + assert backend.get_config_for_mode(mode)[key] == explicit + + +def test_raw_override_preserves_named_connectors_implied_service(): + """Customizing the LMCache payload does not drop the alias's local server dependency.""" + from pathlib import Path + + import yaml + + from srtctl.core.schema import SrtConfig + from srtctl.services.implicit import connector_services + + payload = _CONNECTOR_MAP["lmcache-mp"].transfer_config("agg") + payload["kv_connector_extra_config"]["custom_option"] = True + recipe = yaml.safe_load(Path("examples/features/lmcache-server.yaml").read_text()) + recipe.pop("services") + recipe["roles"]["agg"]["args"]["kv-transfer-config"] = payload + config = SrtConfig.Schema().load(recipe) + + assert [entry.service.type for entry in connector_services(config)] == ["lmcache-server"] + assert json.loads(config.backend.kv_transfer_config("agg")) == payload diff --git a/tests/test_vllm_dp_parallelism.py b/tests/test_vllm_dp_parallelism.py new file mode 100644 index 000000000..e199b2d12 --- /dev/null +++ b/tests/test_vllm_dp_parallelism.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""DP rank placement includes every model-parallel dimension.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from srtctl.backends import VLLMBackend +from srtctl.core.schema import RoleConfig +from srtctl.core.topology import Endpoint + + +@pytest.mark.parametrize("frontend", ["llm-d", "sglang-router", "dynamo"]) +@pytest.mark.parametrize("node_count", [1, 2]) +@pytest.mark.parametrize("tp,pp,pcp", [(1, 1, 2), (2, 1, 2), (1, 2, 2)]) +def test_dp_ranks_own_complete_model_parallel_groups(frontend, node_count, tp, pp, pcp): + replica_size = tp * pp * pcp + backend = VLLMBackend( + dp_launch_mode="per_gpu", + roles={ + "agg": RoleConfig( + args={ + "data-parallel-size": 2 * node_count, + "tensor-parallel-size": tp, + "pipeline-parallel-size": pp, + "prefill-context-parallel-size": pcp, + "data-parallel-external-lb": True, + } + ) + }, + ) + endpoint = Endpoint("agg", 0, tuple(f"node{i}" for i in range(node_count)), frozenset(range(2 * replica_size))) + processes = backend.endpoints_to_processes([endpoint], frontend_type=frontend) + + assert len(processes) == 2 * node_count + assert [p.dp_rank for p in processes] == list(range(2 * node_count)) + for node in endpoint.nodes: + ranks = [p for p in processes if p.node == node] + assert [p.gpu_indices for p in ranks] == [ + frozenset(range(replica_size)), + frozenset(range(replica_size, 2 * replica_size)), + ] + if frontend != "dynamo": + assert all(p.http_port > 0 for p in processes) + assert len({(p.node, p.http_port) for p in processes}) == len(processes) + runtime = MagicMock(model_path=Path("/model"), is_hf_model=False, frontend_port=8000) + with patch("srtctl.core.slurm.get_hostname_ip", return_value="127.0.0.1"): + for process in processes: + command = backend.build_worker_command( + process=process, endpoint_processes=processes, runtime=runtime, frontend_type=frontend + ) + assert command[command.index("--data-parallel-rank") + 1] == str(process.dp_rank) + assert command[command.index("--prefill-context-parallel-size") + 1] == str(pcp) + assert "--headless" not in command + + +@pytest.mark.parametrize( + "dp,pcp,gpu_count,node_count,error", [(2, 4, 2, 1, "require 8 GPUs"), (3, 2, 3, 2, "not divisible")] +) +def test_per_rank_launch_rejects_invalid_pcp_allocation(dp, pcp, gpu_count, node_count, error): + backend = VLLMBackend( + roles={ + "agg": RoleConfig( + args={ + "data-parallel-size": dp, + "prefill-context-parallel-size": pcp, + "data-parallel-external-lb": True, + } + ) + } + ) + nodes = tuple(f"node{i}" for i in range(node_count)) + # The second case has the correct total world size but cannot fit a rank on each node. + endpoint = Endpoint("agg", 0, nodes, frozenset(range(gpu_count))) + with pytest.raises(ValueError, match=error): + backend.endpoints_to_processes([endpoint], frontend_type="llm-d") diff --git a/tests/test_vllm_router_frontend.py b/tests/test_vllm_router_frontend.py index efa4bef22..24019ba62 100644 --- a/tests/test_vllm_router_frontend.py +++ b/tests/test_vllm_router_frontend.py @@ -426,7 +426,7 @@ def test_dp_size_one_is_not_a_distributed_launch_mode() -> None: backend = VLLMBackend(roles={"agg": RoleConfig(args={"data-parallel-size": 1})}) assert backend.find_dp_modes() == [] - assert backend._is_dp_mode("agg") is False + assert backend.is_dp_mode("agg") is False def test_router_validates_pcp_as_part_of_the_vllm_world_size() -> None: