From d33674e14e26a3272dc29145b0109fb1e937a459 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 10 Sep 2026 11:20:38 -0700 Subject: [PATCH 1/5] feat(config): add Qwen3.5 B300 Dynamo-SGLang AgentX recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加 Qwen3.5 NVFP4 B300 Dynamo-SGLang AgentX 配置和配方，并校验合成接受长度与已提交的黄金曲线一致。 --- ...40-b3-k7-mtp-hicache-nightly-20260901.yaml | 157 ++++++++++++++ ...44-b2-k5-mtp-hicache-nightly-20260901.yaml | 136 ++++++++++++ ...p2ep2-highthroughput-nightly-20260901.yaml | 128 +++++++++++ .../agg-b300-tp4-lowlat-nightly-20260901.yaml | 151 +++++++++++++ configs/nvidia-master.yaml | 93 ++++++++ golden_al_distribution/README.md | 7 + golden_al_distribution/README_zh.md | 2 + perf-changelog.yaml | 9 + runners/inject_synthetic_acceptance.py | 105 ++++++++-- runners/launch_b300-dsxe.sh | 14 +- runners/synthetic_injectors/sglang.py | 48 ++++- runners/test_slurm_utils.py | 198 ++++++++++++++++++ 12 files changed, 1020 insertions(+), 28 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml new file mode 100644 index 0000000000..fe3377951a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml @@ -0,0 +1,157 @@ +base: + name: agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901 + model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + precision: fp4 + identity: + model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4 } + container: { image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 } + frameworks: { sglang: nightly-dev-cu13-20260901-07c8f729 } + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 + infra: + nats_max_payload_mb: 8 + frontend: + type: sglang + enable_multiple_frontends: false + backend: + type: sglang + aggregated_environment: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c40-mrr3-k7/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c40-mrr3-k7/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c40-mrr3-k7/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 7 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 8 + speculative-moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 0 + mem-fraction-static: 0.85 + max-running-requests: 3 + pp-max-micro-batch-size: 3 + prefill-max-requests: 3 + cuda-graph-max-bs-decode: 3 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 128 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# Coupled scheduler/graph caps are zipped to preserve one controlled physical +# batch size per variant. The base remains the already measured B3 arm. +zip_override_running_cap: + name: + - agg-b300-tp2-c40-b4-k7-mtp-hicache-nightly-20260901 + - agg-b300-tp2-c40-b7-k7-mtp-hicache-nightly-20260901 + - agg-b300-tp2-c40-b8-k7-mtp-hicache-nightly-20260901 + backend: + sglang_config: + aggregated: + max-running-requests: [4, 7, 8] + pp-max-micro-batch-size: [4, 7, 8] + prefill-max-requests: [4, 7, 8] + cuda-graph-max-bs-decode: [4, 7, 8] + cuda-graph-bs-decode: + - [1, 2, 3, 4] + - [1, 2, 3, 4, 5, 6, 7] + - [1, 2, 3, 4, 5, 6, 7, 8] diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml new file mode 100644 index 0000000000..5a2ada73cf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml @@ -0,0 +1,136 @@ +name: agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + precision: fp4 +identity: + model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4 } + container: { image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 } + frameworks: { sglang: nightly-dev-cu13-20260901-07c8f729 } +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang + enable_multiple_frontends: false +backend: + type: sglang + aggregated_environment: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2-k5/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2-k5/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2-k5/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + speculative-moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 0 + mem-fraction-static: 0.85 + max-running-requests: 2 + pp-max-micro-batch-size: 2 + prefill-max-requests: 2 + cuda-graph-max-bs-decode: 2 + cuda-graph-bs-decode: + - 1 + - 2 + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 128 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml new file mode 100644 index 0000000000..769a27ce7f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml @@ -0,0 +1,128 @@ +name: agg-b300-tp2ep2-highthroughput-nightly-20260901 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + precision: fp4 +identity: + model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4 } + container: { image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 } + frameworks: { sglang: nightly-dev-cu13-20260901-07c8f729 } +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang + enable_multiple_frontends: false +backend: + type: sglang + aggregated_environment: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '2' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/b300-tp2ep2-highthroughput/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/b300-tp2ep2-highthroughput/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/b300-tp2ep2-highthroughput/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 2 + enable-dp-attention: false + enable-dp-lm-head: false + enable-symm-mem: false + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 1048576 + mamba-max-states-per-path: 1 + max-mamba-cache-size: 512 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-prefill-backend: flashinfer + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: flashinfer_trtllm + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + mem-fraction-static: 0.85 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + max-running-requests: 224 + cuda-graph-max-bs: 224 + tokenizer-worker-num: 6 + stream-interval: 50 + scheduler-recv-interval: 10 + allow-auto-truncate: true + enable-hierarchical-cache: true + hicache-ratio: 1.8 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + watchdog-timeout: 1000000 + enable-metrics: true + enable-dynamic-chunking: false + weight-loader-drop-cache-after-load: true +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml new file mode 100644 index 0000000000..eb45ad0ecd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml @@ -0,0 +1,151 @@ +base: + name: agg-b300-tp4-b1-lowlat-nightly-20260901 + model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + precision: fp4 + identity: + model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4 } + container: { image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 } + frameworks: { sglang: nightly-dev-cu13-20260901-07c8f729 } + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: b300 + gpus_per_node: 8 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 4 + infra: + nats_max_payload_mb: 8 + frontend: + type: sglang + enable_multiple_frontends: false + backend: + type: sglang + aggregated_environment: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + MC_FORCE_MNNVL: '1' + NVSHMEM_REMOTE_TRANSPORT: none + MC_TE_METRIC: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/b300-tp4-lowlat/deepgemm-cache + FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/b300-tp4-lowlat/flashinfer-cache + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/b300-tp4-lowlat/sglang-cache + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-dense-tp-size: 4 + enable-dp-attention: false + enable-dp-lm-head: false + moe-a2a-backend: none + load-balance-method: round_robin + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 512 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + speculative-moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + mem-fraction-static: 0.85 + max-running-requests: 1 + pp-max-micro-batch-size: 1 + prefill-max-requests: 1 + cuda-graph-max-bs-decode: 1 + cuda-graph-bs-decode: + - 1 + disable-prefill-cuda-graph: true + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-size: 32 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true + sbatch_directives: + mem: '0' + cpus-per-task: '144' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '4' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# The B2 variant is coupled so scheduler residency, microbatch/prefill caps, +# and decode graph coverage always describe the same physical batch size. +zip_override_batch_cap: + name: + - agg-b300-tp4-b2-lowlat-nightly-20260901 + backend: + sglang_config: + aggregated: + max-running-requests: [2] + pp-max-micro-batch-size: [2] + prefill-max-requests: [2] + cuda-graph-max-bs-decode: [2] + cuda-graph-bs-decode: + - [1, 2] diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index bc1e7f5170..d23ab9f117 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7353,6 +7353,99 @@ qwen3.5-fp4-b300-sglang-agentic-mtp: - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [36, 44, 52] } +# Qwen3.5 NVFP4 B300 Dynamo/SGLang AgentX variants using the pinned +# SGLang image and committed golden acceptance-length curve. +qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto: + image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: cluster:b300-dsxe + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.90 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [64] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 2 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=3.39" + - "CONFIG_FILE=recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [44] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=5.04" + - "CONFIG_FILE=recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml:zip_override_running_cap[2]" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [36] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=5.04" + - "CONFIG_FILE=recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml:zip_override_running_cap[0]" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [40] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=4.41" + - "CONFIG_FILE=recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [4] + num-nodes: 1 + worker: + num-worker: 1 + tp: 4 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=4.80" + - "CONFIG_FILE=recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml:base" + + + # Qwen3.8-Flash-Next NVFP4 AgentX on B300 via SGLang with native NEXTN MTP. # Day-zero recipe; mirrors the B200 arm. TP1 per the cookbook's verified # single-node command: 126 GiB of NVFP4 weights fit on one B300. diff --git a/golden_al_distribution/README.md b/golden_al_distribution/README.md index 061851eee9..e717f460d3 100644 --- a/golden_al_distribution/README.md +++ b/golden_al_distribution/README.md @@ -24,6 +24,13 @@ AL is workload-dependent: a draft model's predictions are easier to accept in so Under the AgentX Guidelines, each model, thinking mode, and draft length has one committed golden AL. Once synthetic acceptance is enabled for a benchmark scenario, a submission may choose any supported draft length, but it may not substitute a different acceptance target. Different models keep their own SPEED-Bench-derived curves. All submissions evaluating the same model and mode use the same curve. +For Qwen3.5 srt-slurm launches, `runners/inject_synthetic_acceptance.py` +resolves the committed curve by the recipe's `speculative-num-steps`, even when +`SYNTHETIC_ACCEPTANCE_LENGTH` is explicitly set. The launch fails if that value +does not match the golden row, if `NUM_SPEC_TOKENS` disagrees with the recipe, +or if `speculative-num-draft-tokens` is not `steps + 1`. The verification token +must not be counted as an additional MTP step. + vLLM supports this through synthetic rejection sampling. For example, an EAGLE3 run can inject the selected YAML value through `synthetic_acceptance_length`: ```bash diff --git a/golden_al_distribution/README_zh.md b/golden_al_distribution/README_zh.md index 79474a4787..04462b74a1 100644 --- a/golden_al_distribution/README_zh.md +++ b/golden_al_distribution/README_zh.md @@ -24,6 +24,8 @@ AL 取决于工作负载：草稿模型的预测在某些领域比其他领域 根据 AgentX 指南，每个模型、思考模式和草稿长度都有一个已提交的黄金 AL。当某个基准场景启用合成接受后，提交可以选择任意受支持的草稿长度，但不能替换为其他接受目标。不同模型保留各自基于 SPEED-Bench 测得的曲线。所有评估同一模型和模式的提交都使用同一条曲线。 +对于 Qwen3.5 的 srt-slurm 启动，`runners/inject_synthetic_acceptance.py` 即使在显式设置 `SYNTHETIC_ACCEPTANCE_LENGTH` 时，也会根据 recipe 中的 `speculative-num-steps` 查询已提交曲线。若该值与黄金表对应行不一致、`NUM_SPEC_TOKENS` 与 recipe 不一致，或 `speculative-num-draft-tokens` 不等于 `steps + 1`，启动都会失败。验证 token 不能额外计为一个 MTP 步数。 + vLLM 通过合成拒绝采样支持这一策略。例如，EAGLE3 运行可以通过 `synthetic_acceptance_length` 注入所选 YAML 值： ```bash diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 792e237fba..335ee530ba 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7138,3 +7138,12 @@ - "Pin vLLM nightly-9ea8f3ffc354901b740f0b31988900897b7221d7, Dynamo 1.5.0.dev20260908, and NVIDIA/srt-slurm d50ee7280c33d469df8708e363e23be2456e94fb." - "Use Dynamo-native MiniMax parsing, EAGLE3 with FlashInfer, FULL_AND_PIECEWISE CUDA graphs, and wait for the OpenAI chat route before AgentX trace replay." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2925 + +- config-keys: + - qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto + scenario-type: + - agentic-coding + description: + - "Add a Qwen3.5 NVFP4 B300 Dynamo-SGLang AgentX configuration with aggregate MTP recipes for TP2/EP2, TP2, and TP4 worker layouts." + - "Use the pinned lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 image and validate each recipe's MTP step count against the committed Qwen3.5 thinking-on golden acceptance-length curve." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/inject_synthetic_acceptance.py b/runners/inject_synthetic_acceptance.py index ab10bb1126..3507538af5 100644 --- a/runners/inject_synthetic_acceptance.py +++ b/runners/inject_synthetic_acceptance.py @@ -26,6 +26,7 @@ python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" "${CONFIG_FILE%%:*}" "$FRAMEWORK" """ +import math import os import sys @@ -41,6 +42,16 @@ "dsv4dsparkprob": "deepseek-v4-pro-0813", } +# Committed golden curves are the authority for AgentX synthetic acceptance. +# Keep this mapping explicit: a model prefix must select one unambiguous curve +# (some models have multiple draft heads / sampling methods). +GOLDEN_AL_REFERENCES = { + "qwen3.5": ( + "qwen3.5_mtp.yaml", + "qwen3.5-397b-a17b-nvfp4", + ), +} + def _log(msg): print(f"[Synthetic AR] {msg}") @@ -87,41 +98,107 @@ def _lookup_al(model_block, num_spec_tokens): return None -def _resolve_al(config_text, injector, ref_yaml): +def _load_reference(ref_yaml, yaml_key, num_spec_tokens): + if not os.path.isfile(ref_yaml): + sys.exit(f"ERROR: golden acceptance reference YAML not found: {ref_yaml}") + + import yaml + + with open(ref_yaml) as f: + data = yaml.safe_load(f) + + model_block = data.get(yaml_key) + if model_block is None: + sys.exit(f'ERROR: model key "{yaml_key}" not found in {ref_yaml}') + + al = _lookup_al(model_block, num_spec_tokens) + if al is None: + sys.exit( + f"ERROR: num_spec_tokens={num_spec_tokens} not found for " + f"{yaml_key} in {ref_yaml}" + ) + return float(al) + + +def _resolve_al(config_text, injector, legacy_ref_yaml): + model_prefix = os.environ.get("MODEL_PREFIX", "").strip() explicit = os.environ.get("SYNTHETIC_ACCEPTANCE_LENGTH", "").strip() + + nst_env = os.environ.get("NUM_SPEC_TOKENS", "").strip() + recipe_spec_tokens = injector.spec_tokens_from_recipe(config_text) + + # Curves listed here are fail closed. In particular, an explicit AL must + # not bypass the recipe's actual speculative step count and silently select + # the next row of the golden table. + reference = GOLDEN_AL_REFERENCES.get(model_prefix) + if reference: + if recipe_spec_tokens is None: + sys.exit( + "ERROR: cannot validate golden AL because the recipe does not " + "declare a speculative-token count" + ) + validator = getattr(injector, "validate_speculative_shape", None) + if validator is not None: + validator(config_text) + if nst_env and int(nst_env) != recipe_spec_tokens: + sys.exit( + f"ERROR: NUM_SPEC_TOKENS={nst_env} disagrees with recipe " + f"speculative-num-steps={recipe_spec_tokens}" + ) + + filename, key = reference + ref_yaml = os.path.join( + os.path.dirname(__file__), "..", "golden_al_distribution", filename + ) + golden_al = _load_reference(ref_yaml, key, recipe_spec_tokens) + if explicit and not math.isclose( + float(explicit), golden_al, rel_tol=0.0, abs_tol=1e-9 + ): + sys.exit( + f"ERROR: explicit SYNTHETIC_ACCEPTANCE_LENGTH={explicit} does " + f"not match golden AL={golden_al:g} for MODEL_PREFIX={model_prefix}, " + f"num_spec_tokens={recipe_spec_tokens} ({ref_yaml})" + ) + _log( + f"Verified golden AL={golden_al:g} from {ref_yaml} " + f"(model={key}, num_spec_tokens={recipe_spec_tokens})" + ) + return golden_al + + # Preserve the legacy reference format for model families that have not yet + # been assigned an unambiguous committed curve above. if explicit: return float(explicit) - if not os.path.isfile(ref_yaml): + if not os.path.isfile(legacy_ref_yaml): sys.exit( "ERROR: SYNTHETIC_ACCEPTANCE_LENGTH not set and reference YAML not " - f"found: {ref_yaml}" + f"found: {legacy_ref_yaml}" ) - import yaml # local import: only needed on the auto-lookup path + import yaml - with open(ref_yaml) as f: + with open(legacy_ref_yaml) as f: data = yaml.safe_load(f) - key = _yaml_key(os.environ.get("MODEL_PREFIX", "")) + key = _yaml_key(model_prefix) model_block = data.get(key) if model_block is None: - sys.exit(f'ERROR: model key "{key}" not found in {ref_yaml}') + sys.exit(f'ERROR: model key "{key}" not found in {legacy_ref_yaml}') - nst_env = os.environ.get("NUM_SPEC_TOKENS", "").strip() - if nst_env: - num_spec_tokens = int(nst_env) - else: - num_spec_tokens = injector.spec_tokens_from_recipe(config_text) or 2 + num_spec_tokens = ( + int(nst_env) if nst_env else (recipe_spec_tokens if recipe_spec_tokens else 2) + ) al = _lookup_al(model_block, num_spec_tokens) if al is None: sys.exit( - f"ERROR: num_spec_tokens={num_spec_tokens} not found for {key} in {ref_yaml}" + f"ERROR: num_spec_tokens={num_spec_tokens} not found for {key} in " + f"{legacy_ref_yaml}" ) _log( - f"Auto-resolved AL={al} from {ref_yaml} " + f"Auto-resolved AL={al} from {legacy_ref_yaml} " f"(model={key}, num_spec_tokens={num_spec_tokens})" ) return float(al) diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index 70254deb65..7dd339d4d4 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -63,8 +63,10 @@ declare -A MODEL_ALIASES=( [nvidia/MiniMax-M3-NVFP4]="MiniMax-M3-NVFP4" [minimax-m3-mxfp8]="MiniMax-M3-MXFP8" [MiniMaxAI/MiniMax-M3-MXFP8]="MiniMax-M3-MXFP8" - [qwen3.5-fp4]="Qwen3.5-397B-A17B-NVFP4-V2" + [qwen3.5-fp4]="Qwen3.5-397B-A17B-NVFP4" + [nvidia/Qwen3.5-397B-A17B-NVFP4]="Qwen3.5-397B-A17B-NVFP4" [qwen3.5-fp8]="Qwen3.5-397B-A17B-FP8" + [qwen3.5-fp4-v2]="Qwen3.5-397B-A17B-NVFP4-V2" [nvidia/Qwen3.5-397B-A17B-NVFP4-V2]="Qwen3.5-397B-A17B-NVFP4-V2" ) @@ -280,10 +282,12 @@ fi # Override the job name in the recipe with the runner name. sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" -if [[ "${EVAL_ONLY:-false}" == "true" ]]; then - python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ - "$CONFIG_PATH" "$FRAMEWORK" || exit 1 -fi +# Throughput recipes opt into synthetic acceptance through the master config. +# Eval-only jobs remove those settings so generated tokens use real target-model +# verification. The injector handles both modes and is a no-op for recipes that +# do not opt in. +python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ + "$CONFIG_PATH" "$FRAMEWORK" || exit 1 # Weights live on node-local MODEL_ROOT, which this login host cannot stat, so # srtctl's preflight model.path check is always skipped. Runtime loading still diff --git a/runners/synthetic_injectors/sglang.py b/runners/synthetic_injectors/sglang.py index d5cd6e0b16..793f3e9a61 100644 --- a/runners/synthetic_injectors/sglang.py +++ b/runners/synthetic_injectors/sglang.py @@ -6,7 +6,12 @@ from . import register _SPEC_STEPS_RE = re.compile(r"(?m)^\s+speculative-num-steps:\s*([0-9]+)\s*$") -_ENV_BLOCK_RE = re.compile(r"(?m)^( (?:aggregated|prefill|decode)_environment:\s*)$") +_SPEC_DRAFT_TOKENS_RE = re.compile( + r"(?m)^\s+speculative-num-draft-tokens:\s*([0-9]+)\s*$" +) +_ENV_BLOCK_RE = re.compile( + r"(?m)^([ \t]+)((?:aggregated|prefill|decode)_environment:\s*)$" +) _SIMULATED_ACCEPTANCE_ENV_RE = re.compile( r"(?m)^[ \t]+SGLANG_SIMULATE_ACC_(?:LEN|METHOD|TOKEN_MODE):[^\n]*(?:\n|$)" ) @@ -14,8 +19,29 @@ def spec_tokens_from_recipe(text): """Read SGLang's speculative step count from the recipe.""" - match = _SPEC_STEPS_RE.search(text) - return int(match.group(1)) if match else None + values = {int(match) for match in _SPEC_STEPS_RE.findall(text)} + if len(values) > 1: + raise ValueError( + f"recipe declares multiple speculative-num-steps values: {sorted(values)}" + ) + return next(iter(values)) if values else None + + +def validate_speculative_shape(text): + """Fail if SGLang's verification window is not steps + one bonus token.""" + steps = spec_tokens_from_recipe(text) + drafts = {int(match) for match in _SPEC_DRAFT_TOKENS_RE.findall(text)} + if steps is None or len(drafts) != 1: + raise ValueError( + "recipe must declare one unambiguous speculative-num-steps and " + "speculative-num-draft-tokens value" + ) + draft_tokens = next(iter(drafts)) + if draft_tokens != steps + 1: + raise ValueError( + f"speculative-num-draft-tokens={draft_tokens} must equal " + f"speculative-num-steps + 1 ({steps + 1})" + ) def rewrite(content, al, log): @@ -23,13 +49,17 @@ def rewrite(content, al, log): if "SGLANG_SIMULATE_ACC_LEN" in content: raise ValueError("recipe already contains SGLANG_SIMULATE_ACC_* variables") - variables = ( - f'\n SGLANG_SIMULATE_ACC_LEN: "{al:g}"' - '\n SGLANG_SIMULATE_ACC_METHOD: "match-expected"' - '\n SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"' - ) + def add_variables(match): + child_indent = match.group(1) + " " + variables = ( + f'\n{child_indent}SGLANG_SIMULATE_ACC_LEN: "{al:g}"' + f'\n{child_indent}SGLANG_SIMULATE_ACC_METHOD: "match-expected"' + f'\n{child_indent}SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token"' + ) + return match.group(0) + variables + rewritten, count = _ENV_BLOCK_RE.subn( - lambda match: match.group(1) + variables, + add_variables, content, ) if count: diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 778b357327..cd33d6b5c8 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -1,5 +1,6 @@ import json import os +import re import runpy import subprocess from pathlib import Path @@ -485,6 +486,203 @@ def test_eval_only_removes_sglang_simulated_acceptance(tmp_path: Path) -> None: assert environment == {"KEEP_ME": "unchanged"} +def test_sglang_throughput_injects_nested_override_environment( + tmp_path: Path, +) -> None: + recipe = tmp_path / "recipe.yaml" + recipe.write_text( + "base:\n" + " backend:\n" + " aggregated_environment:\n" + " KEEP_ME: unchanged\n" + " sglang_config:\n" + " aggregated:\n" + " speculative-num-steps: 5\n" + " speculative-num-draft-tokens: 6\n" + ) + + result = subprocess.run( + ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-sglang"], + env={ + **os.environ, + "SYNTHETIC_ACCEPTANCE": "true", + "SYNTHETIC_ACCEPTANCE_LENGTH": "4.41", + "MODEL_PREFIX": "qwen3.5", + }, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + environment = yaml.safe_load(recipe.read_text())["base"]["backend"][ + "aggregated_environment" + ] + assert environment == { + "KEEP_ME": "unchanged", + "SGLANG_SIMULATE_ACC_LEN": "4.41", + "SGLANG_SIMULATE_ACC_METHOD": "match-expected", + "SGLANG_SIMULATE_ACC_TOKEN_MODE": "real-draft-token", + } + + +def test_qwen35_explicit_acceptance_must_match_recipe_golden_al( + tmp_path: Path, +) -> None: + recipe = tmp_path / "recipe.yaml" + original = ( + "base:\n" + " backend:\n" + " aggregated_environment:\n" + " KEEP_ME: unchanged\n" + " sglang_config:\n" + " aggregated:\n" + " speculative-num-steps: 5\n" + " speculative-num-draft-tokens: 6\n" + ) + recipe.write_text(original) + + result = subprocess.run( + ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-sglang"], + env={ + **os.environ, + "SYNTHETIC_ACCEPTANCE": "true", + "SYNTHETIC_ACCEPTANCE_LENGTH": "4.80", + "MODEL_PREFIX": "qwen3.5", + }, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "does not match golden AL=4.41" in result.stderr + assert "num_spec_tokens=5" in result.stderr + assert recipe.read_text() == original + + +def test_qwen35_auto_lookup_uses_committed_golden_distribution( + tmp_path: Path, +) -> None: + recipe = tmp_path / "recipe.yaml" + recipe.write_text( + "base:\n" + " backend:\n" + " aggregated_environment:\n" + " KEEP_ME: unchanged\n" + " sglang_config:\n" + " aggregated:\n" + " speculative-num-steps: 6\n" + " speculative-num-draft-tokens: 7\n" + ) + + result = subprocess.run( + ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-sglang"], + env={ + **os.environ, + "SYNTHETIC_ACCEPTANCE": "true", + "MODEL_PREFIX": "qwen3.5", + }, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "golden_al_distribution/qwen3.5_mtp.yaml" in result.stdout + assert 'SGLANG_SIMULATE_ACC_LEN: "4.8"' in recipe.read_text() + + +def test_qwen35_acceptance_rejects_wrong_draft_window(tmp_path: Path) -> None: + recipe = tmp_path / "recipe.yaml" + original = ( + "base:\n" + " backend:\n" + " aggregated_environment:\n" + " KEEP_ME: unchanged\n" + " sglang_config:\n" + " aggregated:\n" + " speculative-num-steps: 6\n" + " speculative-num-draft-tokens: 6\n" + ) + recipe.write_text(original) + + result = subprocess.run( + ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-sglang"], + env={ + **os.environ, + "SYNTHETIC_ACCEPTANCE": "true", + "SYNTHETIC_ACCEPTANCE_LENGTH": "4.80", + "MODEL_PREFIX": "qwen3.5", + }, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode != 0 + assert "must equal speculative-num-steps + 1 (7)" in result.stderr + assert recipe.read_text() == original + + +def test_b300_qwen35_pareto_points_match_thinking_on_golden_al() -> None: + master = yaml.safe_load((REPO_ROOT / "configs/nvidia-master.yaml").read_text()) + points = master["qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto"][ + "scenarios" + ]["agentic-coding"][0]["search-space"] + golden = yaml.safe_load( + (REPO_ROOT / "golden_al_distribution/qwen3.5_mtp.yaml").read_text() + )["qwen3.5-397b-a17b-nvfp4"]["thinking_on"] + recipe_root = REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes" + expected = { + 64: (3, 3.39), + 44: (7, 5.04), + 36: (7, 5.04), + 40: (5, 4.41), + 4: (6, 4.80), + } + + actual = {} + for point in points: + settings = dict( + item.split("=", 1) + for item in point["worker"]["additional-settings"] + ) + recipe_rel = settings["CONFIG_FILE"].split(":", 1)[0] + recipe_rel = recipe_rel.removeprefix("recipes/") + recipe_text = (recipe_root / recipe_rel).read_text() + steps = { + int(value) + for value in re.findall(r"speculative-num-steps:\s*(\d+)", recipe_text) + } + drafts = { + int(value) + for value in re.findall( + r"speculative-num-draft-tokens:\s*(\d+)", recipe_text + ) + } + assert len(steps) == 1 + assert len(drafts) == 1 + step = next(iter(steps)) + assert next(iter(drafts)) == step + 1 + al = float(settings["SYNTHETIC_ACCEPTANCE_LENGTH"]) + assert al == float(golden[step]) + actual[point["conc-list"][0]] = (step, al) + + assert actual == expected + + +def test_b300_dsxe_acceptance_driver_is_unconditional_and_fail_closed() -> None: + content = (REPO_ROOT / "runners/launch_b300-dsxe.sh").read_text() + command = 'python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py"' + command_index = content.index(command) + + assert content.rfind("\nfi", 0, command_index) > content.rfind( + "\nif ", 0, command_index + ) + assert "|| exit 1" in content[command_index : command_index + 180] + + def test_sglang_throughput_rejects_existing_simulated_acceptance( tmp_path: Path, ) -> None: From 988d0359abb09e0ed61c8a29488192e741ab53e2 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 10 Sep 2026 11:22:26 -0700 Subject: [PATCH 2/5] chore(changelog): link Qwen3.5 B300 PR #2978 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在性能变更日志中补充 Qwen3.5 B300 PR #2978 链接。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 335ee530ba..71ce69ca9a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7146,4 +7146,4 @@ description: - "Add a Qwen3.5 NVFP4 B300 Dynamo-SGLang AgentX configuration with aggregate MTP recipes for TP2/EP2, TP2, and TP4 worker layouts." - "Use the pinned lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 image and validate each recipe's MTP step count against the committed Qwen3.5 thinking-on golden acceptance-length curve." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2978 From a9dec5361101df45f43eb3ba5e3e569608f47bb8 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Thu, 10 Sep 2026 17:38:05 -0700 Subject: [PATCH 3/5] fix(config): extend Qwen3.5 AgentX Slurm limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 延长 Qwen3.5 AgentX 配方的 Slurm 时间限制。 --- .../agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml | 2 ++ .../agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml | 2 ++ .../agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml | 2 ++ .../b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml index fe3377951a..d755038bc7 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901.yaml @@ -4,6 +4,8 @@ base: path: qwen3.5-fp4 container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 precision: fp4 + slurm: + time_limit: '08:00:00' identity: model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4 } container: { image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 } diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml index 5a2ada73cf..17280c0585 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901.yaml @@ -3,6 +3,8 @@ model: path: qwen3.5-fp4 container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 precision: fp4 +slurm: + time_limit: '08:00:00' identity: model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4 } container: { image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 } diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml index 769a27ce7f..9a1ca27d71 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp2ep2-highthroughput-nightly-20260901.yaml @@ -3,6 +3,8 @@ model: path: qwen3.5-fp4 container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 precision: fp4 +slurm: + time_limit: '08:00:00' identity: model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4 } container: { image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 } diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml index eb45ad0ecd..79df728bd2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/b300-fp4/agentic/agg-b300-tp4-lowlat-nightly-20260901.yaml @@ -4,6 +4,8 @@ base: path: qwen3.5-fp4 container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 precision: fp4 + slurm: + time_limit: '08:00:00' identity: model: { repo: nvidia/Qwen3.5-397B-A17B-NVFP4 } container: { image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 } From 761428402083ded38ae4908e33f7bd6c9b0ef3ee Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Mon, 12 Oct 2026 09:53:37 -0700 Subject: [PATCH 4/5] fix: remove temporary cache environment overrides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移除 Qwen3.5 B300 AgentX 配方中指向 /tmp 的缓存环境变量设置。 --- .../sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml | 6 ------ .../b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml | 6 ------ .../sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml | 3 --- .../b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml | 3 --- 4 files changed, 18 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml index 42f560669b..7c7c6b0237 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml @@ -47,16 +47,10 @@ roles: SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: - /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2-k5/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: - /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2-k5/flashinfer-cache SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_HEALTH_CHECK_TIMEOUT: '1800' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: - /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c44-mrr2-k5/sglang-cache SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml index 7ead4da87e..a2958da15a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml @@ -48,16 +48,10 @@ base: SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: - /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c40-mrr3-k7/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: - /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c40-mrr3-k7/flashinfer-cache SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_HEALTH_CHECK_TIMEOUT: '1800' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: - /tmp/agentx-pareto-v2/main09ec-pr36248-20260829-tp2-fi-trtllm-hicache128-c40-mrr3-k7/sglang-cache SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml index f08a80d86e..1c94fd2dda 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml @@ -47,13 +47,10 @@ roles: SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/b300-tp2ep2-highthroughput/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/b300-tp2ep2-highthroughput/flashinfer-cache SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_HEALTH_CHECK_TIMEOUT: '1800' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/b300-tp2ep2-highthroughput/sglang-cache SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml index 065efc1506..bdf2cf7008 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml @@ -48,13 +48,10 @@ base: SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /tmp/agentx-pareto-v2/b300-tp4-lowlat/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /tmp/agentx-pareto-v2/b300-tp4-lowlat/flashinfer-cache SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_HEALTH_CHECK_TIMEOUT: '1800' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - SGLANG_CACHE_DIR: /tmp/agentx-pareto-v2/b300-tp4-lowlat/sglang-cache SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' From e1269d54b888e283d8fde42f78771fb0e3a98fc8 Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Mon, 12 Oct 2026 19:51:04 -0700 Subject: [PATCH 5/5] feat(agentx): move Qwen3.5 B300 SGLang AgentX to nightly-dev-20261013-7224fce1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Bump the four recipes and the master entry to lmsysorg/sglang:nightly-dev-20261013-7224fce1 and rename the recipe variants to match. - Drop SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE so the NextN draft layer is served at its shipped precision. - Replace aliases this image removed: cuda-graph-max-bs becomes cuda-graph-max-bs-decode (the old spelling is now an ambiguous prefix), and SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1 becomes SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0. - Drop settings with no effect in this image: environment variables it no longer reads (SGLANG_ENABLE_SPEC_V2, SGLANG_ENABLE_FLASHINFER_GEMM, SGLANG_FLASHINFER_FP4_GEMM_BACKEND, SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH), false booleans srt-slurm never passes (including the deprecated enable-dp-attention), and settings whose readers these TP-only aggregate workers never reach (DP-attention scheduler gathers, Mooncake and NVSHMEM variables, load-balance-method, moe-dense-tp-size, and hicache-io-backend: kernel, which page_first_direct rewrites to direct). - Drop the recipe-level cpus-per-task directive, which conflicts with the b300-dsxe cpus-per-gpu default, and run the AgentX client from benchmarks/srt_agentic.sh. 将 Qwen3.5 B300 SGLang AgentX 迁移到 nightly-dev-20261013-7224fce1： - 将四个配方与主配置条目升级到 lmsysorg/sglang:nightly-dev-20261013-7224fce1，并相应重命名配方变体。 - 移除 SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE，使 NextN 草稿层以发布时的精度运行。 - 替换该镜像已移除的别名：cuda-graph-max-bs 改为 cuda-graph-max-bs-decode （旧写法现在是有歧义的前缀），SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1 改为 SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0。 - 移除在该镜像中无效的设置：不再被读取的环境变量（SGLANG_ENABLE_SPEC_V2、 SGLANG_ENABLE_FLASHINFER_GEMM、SGLANG_FLASHINFER_FP4_GEMM_BACKEND、 SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH），srt-slurm 从不传递的 false 布尔参数 （包括已弃用的 enable-dp-attention），以及这些仅 TP 的聚合 worker 不会 触达的设置（DP 注意力调度器 gather、Mooncake 与 NVSHMEM 变量、 load-balance-method、moe-dense-tp-size，以及会被 page_first_direct 改写为 direct 的 hicache-io-backend: kernel）。 - 移除与 b300-dsxe 默认 cpus-per-gpu 冲突的配方级 cpus-per-task 指令，并改为 通过 benchmarks/srt_agentic.sh 运行 AgentX 客户端。 Co-Authored-By: Claude Opus 5.5 --- .../agentx/agg-tp2-b2-mtp5-hicache.yaml | 28 ++++----------- .../agentx/agg-tp2-mtp7-hicache-variants.yaml | 34 +++++-------------- .../agentx/agg-tp2ep2-mtp3-hicache.yaml | 30 ++++------------ .../agentx/agg-tp4-mtp6-hicache-variants.yaml | 30 ++++------------ inferencex-e2e/configs/nvidia-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 4 ++- 6 files changed, 33 insertions(+), 95 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml index 7c7c6b0237..35f63e0e80 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml @@ -1,15 +1,15 @@ schema: 2 -name: agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20260901 +name: agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20261013 model: path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + container: lmsysorg/sglang:nightly-dev-20261013-7224fce1 precision: fp4 slurm: time_limit: '08:00:00' identity: model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4} - container: {image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729} - frameworks: {sglang: nightly-dev-cu13-20260901-07c8f729} + container: {image: lmsysorg/sglang:nightly-dev-20261013-7224fce1} + frameworks: {sglang: nightly-dev-20261013-7224fce1} health_check: max_attempts: 1440 interval_seconds: 10 @@ -35,25 +35,15 @@ roles: NCCL_MNNVL_ENABLE: '1' NCCL_CUMEM_ENABLE: '1' NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' FLASHINFER_DISABLE_VERSION_CHECK: '1' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '1800' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' args: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true @@ -65,11 +55,7 @@ roles: pipeline-parallel-size: 1 data-parallel-size: 1 expert-parallel-size: 1 - moe-dense-tp-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false moe-a2a-backend: none - load-balance-method: round_robin mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 8192 mamba-max-states-per-path: 3 @@ -105,7 +91,6 @@ roles: weight-loader-prefetch-num-threads: 4 enable-hierarchical-cache: true hicache-write-policy: write_back - hicache-io-backend: kernel hicache-mem-layout: page_first_direct hicache-size: 128 enable-metrics: true @@ -113,13 +98,12 @@ roles: disable-attn-tp-gather: true sbatch_directives: mem: '0' - cpus-per-task: '144' srun_options: mem: '0' container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml index a2958da15a..4a7da5a61e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml @@ -1,16 +1,16 @@ schema: 2 base: - name: agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20260901 + name: agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20261013 model: path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + container: lmsysorg/sglang:nightly-dev-20261013-7224fce1 precision: fp4 slurm: time_limit: '08:00:00' identity: model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4} - container: {image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729} - frameworks: {sglang: nightly-dev-cu13-20260901-07c8f729} + container: {image: lmsysorg/sglang:nightly-dev-20261013-7224fce1} + frameworks: {sglang: nightly-dev-20261013-7224fce1} health_check: max_attempts: 1440 interval_seconds: 10 @@ -36,25 +36,15 @@ base: NCCL_MNNVL_ENABLE: '1' NCCL_CUMEM_ENABLE: '1' NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' FLASHINFER_DISABLE_VERSION_CHECK: '1' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '1800' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' args: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true @@ -66,11 +56,7 @@ base: pipeline-parallel-size: 1 data-parallel-size: 1 expert-parallel-size: 1 - moe-dense-tp-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false moe-a2a-backend: none - load-balance-method: round_robin mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 8192 mamba-max-states-per-path: 3 @@ -107,7 +93,6 @@ base: weight-loader-prefetch-num-threads: 4 enable-hierarchical-cache: true hicache-write-policy: write_back - hicache-io-backend: kernel hicache-mem-layout: page_first_direct hicache-size: 128 enable-metrics: true @@ -115,13 +100,12 @@ base: disable-attn-tp-gather: true sbatch_directives: mem: '0' - cpus-per-task: '144' srun_options: mem: '0' container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic @@ -139,9 +123,9 @@ base: # batch size per variant. The base retains the B3 settings. zip_override_running_cap: name: - - agg-b300-tp2-c40-b4-k7-mtp-hicache-nightly-20260901 - - agg-b300-tp2-c40-b7-k7-mtp-hicache-nightly-20260901 - - agg-b300-tp2-c40-b8-k7-mtp-hicache-nightly-20260901 + - agg-b300-tp2-c40-b4-k7-mtp-hicache-nightly-20261013 + - agg-b300-tp2-c40-b7-k7-mtp-hicache-nightly-20261013 + - agg-b300-tp2-c40-b8-k7-mtp-hicache-nightly-20261013 roles: agg: args: diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml index 1c94fd2dda..10f0817d42 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml @@ -1,15 +1,15 @@ schema: 2 -name: agg-b300-tp2ep2-highthroughput-nightly-20260901 +name: agg-b300-tp2ep2-highthroughput-nightly-20261013 model: path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + container: lmsysorg/sglang:nightly-dev-20261013-7224fce1 precision: fp4 slurm: time_limit: '08:00:00' identity: model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4} - container: {image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729} - frameworks: {sglang: nightly-dev-cu13-20260901-07c8f729} + container: {image: lmsysorg/sglang:nightly-dev-20261013-7224fce1} + frameworks: {sglang: nightly-dev-20261013-7224fce1} health_check: max_attempts: 1440 interval_seconds: 10 @@ -35,25 +35,15 @@ roles: NCCL_MNNVL_ENABLE: '1' NCCL_CUMEM_ENABLE: '1' NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' SGLANG_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' FLASHINFER_DISABLE_VERSION_CHECK: '1' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '1800' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' args: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true @@ -65,9 +55,6 @@ roles: pipeline-parallel-size: 1 data-parallel-size: 1 expert-parallel-size: 2 - enable-dp-attention: false - enable-dp-lm-head: false - enable-symm-mem: false mamba-ssm-dtype: bfloat16 mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 1048576 @@ -89,29 +76,26 @@ roles: max-prefill-tokens: 16384 chunked-prefill-size: 16384 max-running-requests: 224 - cuda-graph-max-bs: 224 + cuda-graph-max-bs-decode: 224 tokenizer-worker-num: 6 stream-interval: 50 scheduler-recv-interval: 10 allow-auto-truncate: true enable-hierarchical-cache: true hicache-ratio: 1.8 - hicache-io-backend: kernel hicache-mem-layout: page_first_direct hicache-write-policy: write_back watchdog-timeout: 1000000 enable-metrics: true - enable-dynamic-chunking: false weight-loader-drop-cache-after-load: true sbatch_directives: mem: '0' - cpus-per-task: '144' srun_options: mem: '0' container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml index bdf2cf7008..d1e261a549 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml @@ -1,16 +1,16 @@ schema: 2 base: - name: agg-b300-tp4-b1-lowlat-nightly-20260901 + name: agg-b300-tp4-b1-lowlat-nightly-20261013 model: path: qwen3.5-fp4 - container: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + container: lmsysorg/sglang:nightly-dev-20261013-7224fce1 precision: fp4 slurm: time_limit: '08:00:00' identity: model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4} - container: {image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729} - frameworks: {sglang: nightly-dev-cu13-20260901-07c8f729} + container: {image: lmsysorg/sglang:nightly-dev-20261013-7224fce1} + frameworks: {sglang: nightly-dev-20261013-7224fce1} health_check: max_attempts: 1440 interval_seconds: 10 @@ -36,25 +36,15 @@ base: NCCL_MNNVL_ENABLE: '1' NCCL_CUMEM_ENABLE: '1' NCCL_NVLS_ENABLE: '0' - MC_FORCE_MNNVL: '1' - NVSHMEM_REMOTE_TRANSPORT: none - MC_TE_METRIC: 'true' - SGLANG_ENABLE_SPEC_V2: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' FLASHINFER_DISABLE_VERSION_CHECK: '1' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' SGLANG_HEALTH_CHECK_TIMEOUT: '1800' SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' - SGLANG_SCHEDULER_SKIP_ALL_GATHER: '1' args: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true @@ -66,11 +56,7 @@ base: pipeline-parallel-size: 1 data-parallel-size: 1 expert-parallel-size: 1 - moe-dense-tp-size: 4 - enable-dp-attention: false - enable-dp-lm-head: false moe-a2a-backend: none - load-balance-method: round_robin mamba-radix-cache-strategy: extra_buffer mamba-track-interval: 8192 mamba-max-states-per-path: 3 @@ -105,7 +91,6 @@ base: weight-loader-prefetch-num-threads: 4 enable-hierarchical-cache: true hicache-write-policy: write_back - hicache-io-backend: kernel hicache-mem-layout: page_first_direct hicache-size: 32 enable-metrics: true @@ -113,13 +98,12 @@ base: disable-attn-tp-gather: true sbatch_directives: mem: '0' - cpus-per-task: '144' srun_options: mem: '0' container-remap-root: '' benchmark: type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic @@ -137,7 +121,7 @@ base: # and decode graph coverage always describe the same physical batch size. zip_override_batch_cap: name: - - agg-b300-tp4-b2-lowlat-nightly-20260901 + - agg-b300-tp4-b2-lowlat-nightly-20261013 roles: agg: args: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index d569667ae7..2c18f0fc81 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5251,7 +5251,7 @@ qwen3.5-fp4-b300-sglang-agentic-mtp: # Qwen3.5 NVFP4 B300 Dynamo/SGLang AgentX variants using the pinned # SGLang image and committed golden acceptance-length curve. qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto: - image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 + image: lmsysorg/sglang:nightly-dev-20261013-7224fce1 model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: cluster:b300-dsxe diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index c7d315db08..7e6d1b07d6 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9628,5 +9628,7 @@ - agentic-coding description: - "Add a Qwen3.5 NVFP4 B300 Dynamo-SGLang AgentX configuration with aggregate MTP recipes for TP2/EP2, TP2, and TP4 worker layouts." - - "Use the pinned lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 image and validate each recipe's MTP step count against the committed Qwen3.5 thinking-on golden acceptance-length curve." + - "Use the pinned lmsysorg/sglang:nightly-dev-20261013-7224fce1 image and validate each recipe's MTP step count against the committed Qwen3.5 thinking-on golden acceptance-length curve." + - "Serve the NextN draft layer at its shipped precision: no recipe sets SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE." + - "Replace aliases removed in this image (cuda-graph-max-bs becomes cuda-graph-max-bs-decode; SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1 becomes SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0) and drop settings that have no effect in it." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2978