feat(qwen35): tp2 phase 2a

.github/workflows/ci.yml

@@ -319,3 +319,71 @@ jobs:
           CUDA_PATH: /usr/local/cuda
           PEGAINFER_CUDA_SM: "80"
           PEGAINFER_NVCC_JOBS: "2"
+
+  qwen35-cuda:
+    if: github.event_name == 'push' || github.event.pull_request.draft == false
+    name: Qwen3.5 CUDA ${{ matrix.gate }} (sm_80)
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - gate: Compile
+            cargo_command: check
+            trailing_args: ""
+          - gate: Clippy
+            cargo_command: clippy
+            trailing_args: "-- -D warnings"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+          submodules: recursive
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: nightly-2026-07-10
+          components: clippy
+
+      - name: Setup sccache
+        uses: mozilla-actions/sccache-action@v0.0.10
+        with:
+          version: "v0.16.0"
+
+      - name: Install CUDA toolkit
+        uses: Jimver/cuda-toolkit@v0.2.35
+        with:
+          cuda: "13.0.2"
+          linux-local-args: '["--toolkit"]'
+          method: network
+          sub-packages: '["nvcc", "nvrtc-dev", "cudart-dev"]'
+          non-cuda-sub-packages: '["libcublas-dev", "libcurand-dev"]'
+
+      - name: Install Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.10"
+
+      - name: Install Qwen3.5 build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y protobuf-compiler libibverbs-dev
+          python -m pip install "triton==3.7.1"
+
+      - name: Run Qwen3.5 ${{ matrix.gate }}
+        run: >-
+          cargo ${{ matrix.cargo_command }} --release --locked
+          -p pegainfer-qwen35 --features qwen35 --all-targets
+          ${{ matrix.trailing_args }}
+        env:
+          CUDA_PATH: /usr/local/cuda
+          PEGAINFER_CUDA_SM: "80"
+          PEGAINFER_NVCC_JOBS: "2"
+          PEGAINFER_TRITON_PYTHON: python

docs/benchmarks/qwen35-tp2-phase2a-multiturn.json

@@ -0,0 +1 @@
+{"date":"2026-08-19T14:24:36Z","mode":"multi_turn","backend":"openai-chat","model_id":"qwen35-tp2-phase2a","num_conversations":12,"max_model_len":2048,"turns_per_conversation":3,"multi_turn_concurrency":4,"inter_turn_delay_ms":0,"server_commit":"8829189992d9290f1aa128fb00950dd854f43211","client_commit":"2b7fcbf52782f8729fd6ce6c9ab803617d72897b","gpu":"2x_RTX_3090_sm86","tp_size":"2","server_max_batch":"2","server_max_prefill_tokens":"64","duration":13.864881841999999,"conversations_completed":12,"conversations_failed":0,"avg_turns_completed":3.6666666666666665,"avg_conversation_duration_ms":10451.412879833333,"completed":44,"failed":0,"total_input_tokens":12496,"total_output_tokens":1056,"request_throughput":3.173485392909272,"input_throughput":901.2698515862332,"output_throughput":76.16364942982253,"total_token_throughput":977.4335010160559,"mean_ttft_ms":727.5413035227273,"median_ttft_ms":727.6886789999999,"std_ttft_ms":146.66137831986737,"p50_ttft_ms":727.6886789999999,"p90_ttft_ms":847.6425581,"p99_ttft_ms":899.2532077699999,"mean_tpot_ms":21.70376772332016,"median_tpot_ms":21.55133260869565,"std_tpot_ms":0.8264917019729615,"p50_tpot_ms":21.55133260869565,"p90_tpot_ms":22.558429852173912,"p99_tpot_ms":22.933247999565218,"mean_itl_ms":19.96713433909089,"median_itl_ms":20.5969695,"std_itl_ms":6.481995568729013,"p50_itl_ms":20.5969695,"p90_itl_ms":26.920911500000003,"p99_itl_ms":30.078894709999997,"mean_e2el_ms":1226.7279611590914,"median_e2el_ms":1222.6157295,"std_e2el_ms":153.48850861286388,"p50_e2el_ms":1222.6157295,"p90_e2el_ms":1365.4634053,"p99_e2el_ms":1406.7637406399997,"per_turn_metrics":[{"turn_index":0,"num_samples":12,"completed":12,"failed":0,"total_input_tokens":1824,"total_output_tokens":288,"request_throughput":0.8654960162479833,"input_throughput":131.55539446969345,"output_throughput":20.7719043899516,"total_token_throughput":152.32729885964505,"mean_ttft_ms":583.7918206666667,"median_ttft_ms":657.463881,"std_ttft_ms":198.93882849514316,"p50_ttft_ms":657.463881,"p90_ttft_ms":691.2390482,"p99_ttft_ms":742.9091798999999,"mean_tpot_ms":21.169481184782605,"median_tpot_ms":21.11208156521739,"std_tpot_ms":0.1470605595878701,"p50_tpot_ms":21.11208156521739,"p90_tpot_ms":21.43137102173913,"p99_tpot_ms":21.507665196521735,"mean_itl_ms":19.47549855000002,"median_itl_ms":20.379788,"std_itl_ms":6.2034251783074525,"p50_itl_ms":20.379788,"p90_itl_ms":21.269388,"p99_itl_ms":30.728750939999998,"mean_e2el_ms":1070.6898879166667,"median_e2el_ms":1147.035556,"std_e2el_ms":199.1779687190421,"p50_e2el_ms":1147.035556,"p90_e2el_ms":1175.5356078000002,"p99_e2el_ms":1228.5809922},{"turn_index":1,"num_samples":12,"completed":12,"failed":0,"total_input_tokens":2880,"total_output_tokens":288,"request_throughput":0.8654960162479833,"input_throughput":207.719043899516,"output_throughput":20.7719043899516,"total_token_throughput":228.49094828946758,"mean_ttft_ms":714.34124925,"median_ttft_ms":720.4639175000001,"std_ttft_ms":16.123405097003147,"p50_ttft_ms":720.4639175000001,"p90_ttft_ms":722.1488714999999,"p99_ttft_ms":732.58807688,"mean_tpot_ms":21.50762221376812,"median_tpot_ms":21.512210391304347,"std_tpot_ms":0.05180367387233731,"p50_tpot_ms":21.512210391304347,"p90_tpot_ms":21.557149665217388,"p99_tpot_ms":21.592176384782608,"mean_itl_ms":19.786716913333336,"median_itl_ms":20.530554,"std_itl_ms":6.297680823403235,"p50_itl_ms":20.530554,"p90_itl_ms":26.572883,"p99_itl_ms":30.4137518,"mean_e2el_ms":1209.0165601666667,"median_e2el_ms":1214.526274,"std_e2el_ms":15.950441454258286,"p50_e2el_ms":1214.526274,"p90_e2el_ms":1216.6144174,"p99_e2el_ms":1227.28499475},{"turn_index":2,"num_samples":9,"completed":9,"failed":0,"total_input_tokens":2952,"total_output_tokens":216,"request_throughput":0.6491220121859875,"input_throughput":212.91201999700388,"output_throughput":15.578928292463699,"total_token_throughput":228.49094828946758,"mean_ttft_ms":778.062378,"median_ttft_ms":783.74919,"std_ttft_ms":19.224187823957255,"p50_ttft_ms":783.74919,"p90_ttft_ms":789.8680276000001,"p99_ttft_ms":805.37482096,"mean_tpot_ms":22.044310932367154,"median_tpot_ms":21.928995999999998,"std_tpot_ms":0.21255355180988741,"p50_tpot_ms":21.928995999999998,"p90_tpot_ms":22.314300469565218,"p99_tpot_ms":22.48564071652174,"mean_itl_ms":20.280475453333345,"median_itl_ms":20.668806,"std_itl_ms":6.545678930929194,"p50_itl_ms":20.668806,"p90_itl_ms":26.9185278,"p99_itl_ms":29.846178479999995,"mean_e2el_ms":1285.0815294444446,"median_e2el_ms":1287.6416179999999,"std_e2el_ms":17.447565570324695,"p50_e2el_ms":1287.6416179999999,"p90_e2el_ms":1303.2953988,"p99_e2el_ms":1310.6299300800001},{"turn_index":3,"num_samples":8,"completed":8,"failed":0,"total_input_tokens":3328,"total_output_tokens":192,"request_throughput":0.5769973441653222,"input_throughput":240.03089517277405,"output_throughput":13.847936259967733,"total_token_throughput":253.87883143274178,"mean_ttft_ms":845.3753072500001,"median_ttft_ms":846.2088815,"std_ttft_ms":13.52094063421677,"p50_ttft_ms":846.2088815,"p90_ttft_ms":855.3806129000001,"p99_ttft_ms":871.3374689899999,"mean_tpot_ms":22.59411058695652,"median_tpot_ms":22.542907565217387,"std_tpot_ms":0.1418984913926098,"p50_tpot_ms":22.542907565217387,"p90_tpot_ms":22.768807678260867,"p99_tpot_ms":22.91283586782609,"mean_itl_ms":20.786269600000008,"median_itl_ms":20.8269005,"std_itl_ms":6.734698925762719,"p50_itl_ms":20.8269005,"p90_itl_ms":27.4097134,"p99_itl_ms":29.42091591,"mean_e2el_ms":1365.03985075,"median_e2el_ms":1364.992177,"std_e2el_ms":14.886185612840757,"p50_e2el_ms":1364.992177,"p90_e2el_ms":1379.5870000000002,"p99_e2el_ms":1393.6527202},{"turn_index":4,"num_samples":3,"completed":3,"failed":0,"total_input_tokens":1512,"total_output_tokens":72,"request_throughput":0.21637400406199583,"input_throughput":109.05249804724589,"output_throughput":5.1929760974879,"total_token_throughput":114.24547414473379,"mean_ttft_ms":889.5522186666667,"median_ttft_ms":883.5537770000001,"std_ttft_ms":15.724934853616045,"p50_ttft_ms":883.5537770000001,"p90_ttft_ms":905.5880658,"p99_ttft_ms":910.54578078,"mean_tpot_ms":21.229618652173915,"median_tpot_ms":22.913742956521734,"std_tpot_ms":2.3978735945256315,"p50_tpot_ms":22.913742956521734,"p90_tpot_ms":22.932007860869565,"p99_tpot_ms":22.936117464347824,"mean_itl_ms":19.530963160000006,"median_itl_ms":20.905244,"std_itl_ms":7.1278942722312735,"p50_itl_ms":20.905244,"p90_itl_ms":27.1184224,"p99_itl_ms":28.359043100000008,"mean_e2el_ms":1377.8334476666664,"median_e2el_ms":1401.022329,"std_e2el_ms":40.12772579823275,"p50_e2el_ms":1401.022329,"p90_e2el_ms":1409.0804506,"p99_e2el_ms":1410.8935279599998}]}

docs/benchmarks/qwen35-tp2-phase2a-multiturn.md

@@ -0,0 +1,256 @@
+# Qwen3.5 Phase 2A TP2 Multi-Turn Serving Gate
+
+> **TL;DR:** A real 2x RTX 3090 Qwen3.5 TP2 server completed 12/12 dependent conversations and 44/44 measured turns at client concurrency 4 over server capacity 2, then admitted and completed another 4/4 conversations and 8/8 turns without restart; all requests produced the configured token counts, history grew on every turn, shutdown was clean, and both GPUs returned to idle.
+
+This is a correctness and lifecycle gate for Phase 2A. It is not a comparative
+performance claim. The run was captured on 2026-08-19 UTC.
+
+## Pass Criteria
+
+The run passes only when all of the following hold:
+
+1. The dry run builds 12 conversations with 2-5 turns and no server traffic.
+2. The primary run completes 12/12 conversations and 44/44 measured turns with zero failures.
+3. Every measured turn returns exactly 24 output tokens, for 1,056 output tokens total.
+4. Per-turn sample counts prove mixed conversation lengths, and prompt accounting grows with carried chat history.
+5. Client concurrency 4 completes against `max_batch=2` without deadlock or rank divergence.
+6. Without restarting the server, a second probe completes 4/4 newly admitted conversations and 8/8 turns with zero failures.
+7. Graceful shutdown exits the scheduler and releases both GPUs.
+
+All seven criteria passed.
+
+## Reproduction Pins
+
+| Component | Value |
+| --- | --- |
+| PegaInfer server commit | `8829189992d9290f1aa128fb00950dd854f43211` |
+| PegaInfer binary SHA-256 | `97ab83bbf666add4e4b0854c1bea77688279ca27dc87bc15641fb7d7e29cd9af` |
+| vLLM client commit | `2b7fcbf52782f8729fd6ce6c9ab803617d72897b` |
+| `vllm-bench` version | `0.1.0` |
+| `vllm-bench` binary SHA-256 | `8131ed513d22da21186eb7ccba06dfb6d0c8657624bb72a733948841a5dd1ffe` |
+| GPUs | 2x NVIDIA RTX 3090, SM86, 24 GiB each |
+| Model fixture | [`Qwen/Qwen3.5-4B`](https://huggingface.co/Qwen/Qwen3.5-4B/tree/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a), revision `851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a` |
+| `config.json` SHA-256 | `ddc63e1c717afa86c865bb5e01313d89d72bb53b97ad4a8a03ba8510c0621670` |
+| `tokenizer.json` SHA-256 | `5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42` |
+| `model.safetensors.index.json` SHA-256 | `cf3f798ee02ba45f9622aa8892a47369ab667d0afbf154ee7c2212de42e6302d` |
+| `model.safetensors-00001-of-00002.safetensors` SHA-256 | `26a93f066e1916adb13453dae5a0c707c0fbc71299ed98779571a907b8e74c61` |
+| `model.safetensors-00002-of-00002.safetensors` SHA-256 | `cb544bd9bfae93dc59b0f22b292f5933573854a7f9b97835c67060d7d910e188` |
+| TP/runtime | TP=2, eager, CUDA Graph disabled, `max_batch=2`, `max_prefill_tokens=64`, scheduler policy `off` |
+
+The repository's older pinned vLLM revision predates the Rust `vllm-bench`
+workspace. The client was therefore built from the exact maintained upstream
+commit above. Source inspection confirmed that `--multi-turn` runs turns
+sequentially within each conversation, appends each assistant response, and
+sends the accumulated message history on the next turn. A separate sequential
+chat driver was not needed.
+
+Set these paths before running the commands. `MODEL_DIR` must point to the
+pinned public revision above and match the complete config, tokenizer, index,
+and weight-shard hash manifest; no private absolute model path is part of the
+reproduction contract.
+
+```bash
+MODEL_DIR=/path/to/public/Qwen3.5-4B
+VLLM_SRC=/tmp/vllm-phase2a-bench-src
+VLLM_BENCH="$VLLM_SRC/rust/target/release/vllm-bench"
+RESULT_DIR=/tmp/qwen35-tp2-phase2a-results
+```
+
+Build the pinned client:
+
+```bash
+git clone --filter=blob:none --no-checkout https://github.com/vllm-project/vllm.git "$VLLM_SRC"
+git -C "$VLLM_SRC" fetch --depth 1 origin 2b7fcbf52782f8729fd6ce6c9ab803617d72897b
+git -C "$VLLM_SRC" checkout --detach 2b7fcbf52782f8729fd6ce6c9ab803617d72897b
+cargo build --release -p vllm-bench --manifest-path "$VLLM_SRC/rust/Cargo.toml"
+"$VLLM_BENCH" --version
+```
+
+Build and start PegaInfer from the server commit above:
+
+```bash
+PEGAINFER_CUDA_SM=86 \
+PEGAINFER_TRITON_PYTHON="$PWD/.venv/bin/python" \
+PROTOC="$PWD/.venv/protoc-root/usr/bin/protoc" \
+PROTOC_INCLUDE="$PWD/.venv/protoc-root/usr/include" \
+LD_LIBRARY_PATH="$PWD/.venv/protoc-root/usr/lib/x86_64-linux-gnu" \
+cargo build --offline --release --locked \
+  -p pegainfer-server --no-default-features --features qwen35
+```
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1 \
+PEGAINFER_CUDA_SM=86 \
+PEGAINFER_TRITON_PYTHON="$PWD/.venv/bin/python" \
+LD_LIBRARY_PATH="$PWD/.venv/lib/python3.10/site-packages/nvidia/nccl/lib" \
+RUST_LOG=info \
+target/release/pegainfer \
+  --model-path "$MODEL_DIR" \
+  --served-model-name qwen35-tp2-phase2a \
+  --port 18080 \
+  --tp-size 2 \
+  --cuda-graph=false \
+  --max-batch 2 \
+  --max-prefill-tokens 64 \
+  --qwen35-scheduler-policy off
+```
+
+## Workload Commands
+
+The dry run fixes the generated dataset before server traffic:
+
+```bash
+"$VLLM_BENCH" \
+  --backend openai-chat \
+  --model qwen35-tp2-phase2a \
+  --tokenizer "$MODEL_DIR" \
+  --dataset-name random \
+  --multi-turn \
+  --multi-turn-min-turns 2 \
+  --multi-turn-max-turns 5 \
+  --random-input-len 128 \
+  --per-turn-input-len 64 \
+  --random-output-len 24 \
+  --num-prompts 12 \
+  --multi-turn-concurrency 4 \
+  --max-model-len 2048 \
+  --seed 446 \
+  --dry-run
+```
+
+It produced 12 conversations, 44 turns, and 3,584 user-message tokens.
+The measured primary run used the same seed and shape:
+
+```bash
+mkdir -p "$RESULT_DIR"
+"$VLLM_BENCH" \
+  --backend openai-chat \
+  --base-url http://127.0.0.1:18080 \
+  --model qwen35-tp2-phase2a \
+  --tokenizer "$MODEL_DIR" \
+  --dataset-name random \
+  --multi-turn \
+  --multi-turn-min-turns 2 \
+  --multi-turn-max-turns 5 \
+  --random-input-len 128 \
+  --per-turn-input-len 64 \
+  --random-output-len 24 \
+  --num-prompts 12 \
+  --multi-turn-concurrency 4 \
+  --max-model-len 2048 \
+  --seed 446 \
+  --ignore-eos \
+  --temperature 0 \
+  --ready-check-timeout-sec 30 \
+  --percentile-metrics ttft,tpot,itl,e2el \
+  --metric-percentiles 50,90,99 \
+  --save-result \
+  --save-detailed \
+  --result-dir "$RESULT_DIR" \
+  --result-filename qwen35-tp2-phase2a-multiturn.json \
+  --metadata \
+    server_commit=8829189992d9290f1aa128fb00950dd854f43211 \
+    client_commit=2b7fcbf52782f8729fd6ce6c9ab803617d72897b \
+    gpu=2x_RTX_3090_sm86 \
+    tp_size=2 \
+    server_max_batch=2 \
+    server_max_prefill_tokens=64
+```
+
+The post-workload probe was run immediately afterward, without restarting or
+reloading the server:
+
+```bash
+"$VLLM_BENCH" \
+  --backend openai-chat \
+  --base-url http://127.0.0.1:18080 \
+  --model qwen35-tp2-phase2a \
+  --tokenizer "$MODEL_DIR" \
+  --dataset-name random \
+  --multi-turn \
+  --multi-turn-num-turns 2 \
+  --random-input-len 32 \
+  --per-turn-input-len 16 \
+  --random-output-len 8 \
+  --num-prompts 4 \
+  --multi-turn-concurrency 4 \
+  --max-model-len 512 \
+  --seed 447 \
+  --ignore-eos \
+  --temperature 0 \
+  --ready-check-timeout-sec 30 \
+  --percentile-metrics ttft,tpot,itl,e2el \
+  --metric-percentiles 50,90,99 \
+  --save-result \
+  --result-dir "$RESULT_DIR" \
+  --result-filename qwen35-tp2-phase2a-post-readmission.json \
+  --metadata \
+    purpose=post_workload_readmission \
+    server_commit=8829189992d9290f1aa128fb00950dd854f43211 \
+    client_commit=2b7fcbf52782f8729fd6ce6c9ab803617d72897b \
+    tp_size=2 \
+    server_max_batch=2
+```
+
+## Results
+
+| Metric | Primary run | Post-workload probe |
+| --- | ---: | ---: |
+| Conversations completed/failed | 12 / 0 | 4 / 0 |
+| Measured turns completed/failed | 44 / 0 | 8 / 0 |
+| Readiness requests, unmeasured | 1 | 1 |
+| Client concurrency | 4 | 4 |
+| Duration | 13.865 s | 0.786 s |
+| Input tokens | 12,496 | 416 |
+| Output tokens | 1,056 | 64 |
+| Request throughput | 3.173 req/s | 10.185 req/s |
+| Output throughput | 76.164 tok/s | 81.475 tok/s |
+
+Primary latency percentiles:
+
+| Metric | Mean | p50 | p90 | p99 |
+| --- | ---: | ---: | ---: | ---: |
+| TTFT | 727.54 ms | 727.69 ms | 847.64 ms | 899.25 ms |
+| TPOT | 21.70 ms | 21.55 ms | 22.56 ms | 22.93 ms |
+| ITL | 19.97 ms | 20.60 ms | 26.92 ms | 30.08 ms |
+| E2EL | 1226.73 ms | 1222.62 ms | 1365.46 ms | 1406.76 ms |
+
+Turn-level evidence from the primary run:
+
+| Turn | Conversations reaching turn | Client-accounted input per request | Server prompt-token range | Output per request |
+| ---: | ---: | ---: | ---: | ---: |
+| 1 | 12 | 152 | 138-140 | 24 |
+| 2 | 12 | 240 | 212-214 | 24 |
+| 3 | 9 | 328 | 286-288 | 24 |
+| 4 | 8 | 416 | 360-362 | 24 |
+| 5 | 3 | 504 | 435 | 24 |
+
+The `[12, 12, 9, 8, 3]` sample vector proves that conversations had mixed 2-5
+turn lengths. Both the client accounting and the server's rendered prompt
+tokens increase per turn, which proves that later requests carried earlier
+user and assistant messages instead of flattening the dataset to its first
+turn. Client and server token counts differ because the server applies the chat
+template and tokenizer to the accumulated messages.
+
+This client revision still writes the default value `turns_per_conversation=3`
+in the top-level JSON even when min/max turn sampling is enabled. The measured
+per-turn sample vector and `avg_turns_completed=3.6667` are the authoritative
+values for this variable-length run.
+
+The client emitted a warning that `--ignore-eos` can interact with multi-turn
+output limits. In this run, every one of the 44 server responses logged
+`output_tokens=24` with `finish_reason=length`; the saved result totals exactly
+`44 * 24 = 1,056` output tokens. The post-workload probe similarly totals
+`8 * 8 = 64`.
+
+The primary result is stored in
+[qwen35-tp2-phase2a-multiturn.json](qwen35-tp2-phase2a-multiturn.json), SHA-256
+`9247a35bc3f5475ec59397ddd2dd3461b9c033e1d27ef25f64c7517da5398676`.
+The no-restart readmission result is stored in
+[qwen35-tp2-phase2a-post-readmission.json](qwen35-tp2-phase2a-post-readmission.json),
+SHA-256 `a60890446d3bb08f83da37bf7c14c2a9063eccbdc480ae3cbb028ccd52ce2573`.
+
+After the second run, Ctrl-C produced `scheduler: all handles dropped, exiting`
+and process exit code 0. A subsequent `nvidia-smi` showed both TP devices at 1
+MiB used, 24,126 MiB free, and 0% utilization, confirming process-level resource
+release.