diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml new file mode 100644 index 0000000000..35f63e0e80 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml @@ -0,0 +1,118 @@ +schema: 2 +name: agg-b300-tp2-c44-b2-k5-mtp-hicache-nightly-20261013 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-20261013-7224fce1 + precision: fp4 +slurm: + time_limit: '08:00:00' +identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4} + container: {image: lmsysorg/sglang:nightly-dev-20261013-7224fce1} + frameworks: {sglang: nightly-dev-20261013-7224fce1} +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: sglang-router + enable_multiple_frontends: false +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_MOE_NVFP4_DISPATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-a2a-backend: none + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 5 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 6 + speculative-moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 0 + mem-fraction-static: 0.85 + max-running-requests: 2 + pp-max-micro-batch-size: 2 + prefill-max-requests: 2 + cuda-graph-max-bs-decode: 2 + cuda-graph-bs-decode: + - 1 + - 2 + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-mem-layout: page_first_direct + hicache-size: 128 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml new file mode 100644 index 0000000000..4a7da5a61e --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml @@ -0,0 +1,139 @@ +schema: 2 +base: + name: agg-b300-tp2-c40-b3-k7-mtp-hicache-nightly-20261013 + model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-20261013-7224fce1 + precision: fp4 + slurm: + time_limit: '08:00:00' + identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4} + container: {image: lmsysorg/sglang:nightly-dev-20261013-7224fce1} + frameworks: {sglang: nightly-dev-20261013-7224fce1} + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang-router + enable_multiple_frontends: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_MOE_NVFP4_DISPATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-a2a-backend: none + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 1536 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 7 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 8 + speculative-moe-runner-backend: flashinfer_trtllm + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + prefill-decode-interval: 0 + mem-fraction-static: 0.85 + max-running-requests: 3 + pp-max-micro-batch-size: 3 + prefill-max-requests: 3 + cuda-graph-max-bs-decode: 3 + cuda-graph-bs-decode: + - 1 + - 2 + - 3 + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-mem-layout: page_first_direct + hicache-size: 128 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true + sbatch_directives: + mem: '0' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# Coupled scheduler/graph caps are zipped to preserve one controlled physical +# batch size per variant. The base retains the B3 settings. +zip_override_running_cap: + name: + - agg-b300-tp2-c40-b4-k7-mtp-hicache-nightly-20261013 + - agg-b300-tp2-c40-b7-k7-mtp-hicache-nightly-20261013 + - agg-b300-tp2-c40-b8-k7-mtp-hicache-nightly-20261013 + roles: + agg: + args: + max-running-requests: [4, 7, 8] + pp-max-micro-batch-size: [4, 7, 8] + prefill-max-requests: [4, 7, 8] + cuda-graph-max-bs-decode: [4, 7, 8] + cuda-graph-bs-decode: + - [1, 2, 3, 4] + - [1, 2, 3, 4, 5, 6, 7] + - [1, 2, 3, 4, 5, 6, 7, 8] diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml new file mode 100644 index 0000000000..10f0817d42 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml @@ -0,0 +1,110 @@ +schema: 2 +name: agg-b300-tp2ep2-highthroughput-nightly-20261013 +model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-20261013-7224fce1 + precision: fp4 +slurm: + time_limit: '08:00:00' +identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4} + container: {image: lmsysorg/sglang:nightly-dev-20261013-7224fce1} + frameworks: {sglang: nightly-dev-20261013-7224fce1} +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: b300 + gpus_per_node: 8 +frontend: + type: sglang-router + enable_multiple_frontends: false +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 2 + env: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '2' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_MOE_NVFP4_DISPATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 2 + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 1048576 + mamba-max-states-per-path: 1 + max-mamba-cache-size: 512 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-prefill-backend: flashinfer + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: flashinfer_trtllm + enable-linear-replayssm-spec: true + linear-replayssm-cache-len: 8 + mem-fraction-static: 0.85 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + max-running-requests: 224 + cuda-graph-max-bs-decode: 224 + tokenizer-worker-num: 6 + stream-interval: 50 + scheduler-recv-interval: 10 + allow-auto-truncate: true + enable-hierarchical-cache: true + hicache-ratio: 1.8 + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + watchdog-timeout: 1000000 + enable-metrics: true + weight-loader-drop-cache-after-load: true +sbatch_directives: + mem: '0' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml new file mode 100644 index 0000000000..d1e261a549 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml @@ -0,0 +1,133 @@ +schema: 2 +base: + name: agg-b300-tp4-b1-lowlat-nightly-20261013 + model: + path: qwen3.5-fp4 + container: lmsysorg/sglang:nightly-dev-20261013-7224fce1 + precision: fp4 + slurm: + time_limit: '08:00:00' + identity: + model: {repo: nvidia/Qwen3.5-397B-A17B-NVFP4} + container: {image: lmsysorg/sglang:nightly-dev-20261013-7224fce1} + frameworks: {sglang: nightly-dev-20261013-7224fce1} + health_check: + max_attempts: 1440 + interval_seconds: 10 + resources: + gpu_type: b300 + gpus_per_node: 8 + frontend: + type: sglang-router + enable_multiple_frontends: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + env: + SGLANG_TRTLLM_MHA_DECODE_SEQ_LEN_SPLITS: '1' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NO_COLOR: '1' + PYTHONUNBUFFERED: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_NVLS_ENABLE: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_MOE_NVFP4_DISPATCH: '1' + FLASHINFER_DISABLE_VERSION_CHECK: '1' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_FLASHINFER_AUTOTUNE_CACHE: '1' + SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK: '1' + args: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + moe-a2a-backend: none + mamba-radix-cache-strategy: extra_buffer + mamba-track-interval: 8192 + mamba-max-states-per-path: 3 + mamba-ssm-dtype: bfloat16 + max-mamba-cache-size: 512 + context-length: 262144 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_cutedsl + linear-attn-decode-backend: triton + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 6 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + speculative-moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: none + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + mem-fraction-static: 0.85 + max-running-requests: 1 + pp-max-micro-batch-size: 1 + prefill-max-requests: 1 + cuda-graph-max-bs-decode: 1 + cuda-graph-bs-decode: + - 1 + disable-prefill-cuda-graph: true + stream-interval: 20 + decode-log-interval: 10 + watchdog-timeout: 1000000 + weight-loader-prefetch-checkpoints: true + weight-loader-prefetch-num-threads: 4 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-mem-layout: page_first_direct + hicache-size: 32 + enable-metrics: true + enable-linear-replayssm-spec: true + disable-attn-tp-gather: true + sbatch_directives: + mem: '0' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '4' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + +# The B2 variant is coupled so scheduler residency, microbatch/prefill caps, +# and decode graph coverage always describe the same physical batch size. +zip_override_batch_cap: + name: + - agg-b300-tp4-b2-lowlat-nightly-20261013 + roles: + agg: + args: + max-running-requests: [2] + pp-max-micro-batch-size: [2] + prefill-max-requests: [2] + cuda-graph-max-bs-decode: [2] + cuda-graph-bs-decode: + - [1, 2] diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 29f8b7c55c..2c18f0fc81 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5248,6 +5248,87 @@ qwen3.5-fp4-b300-sglang-agentic-mtp: - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml } - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [36, 44, 52], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/b300-fp4-mtp/agentic.yaml } +# Qwen3.5 NVFP4 B300 Dynamo/SGLang AgentX variants using the pinned +# SGLang image and committed golden acceptance-length curve. +qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto: + image: lmsysorg/sglang:nightly-dev-20261013-7224fce1 + model: nvidia/Qwen3.5-397B-A17B-NVFP4 + model-prefix: qwen3.5 + runner: cluster:b300-dsxe + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.90 + search-space: + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [64] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 2 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2ep2-mtp3-hicache.yaml" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [44] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml:zip_override_running_cap[2]" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [36] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-mtp7-hicache-variants.yaml:zip_override_running_cap[0]" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [40] + num-nodes: 1 + worker: + num-worker: 1 + tp: 2 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp2-b2-mtp5-hicache.yaml" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [4] + num-nodes: 1 + worker: + num-worker: 1 + tp: 4 + pp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/qwen3.5/sglang/b300-fp4/agentx/agg-tp4-mtp6-hicache-variants.yaml:base" + # Qwen3.8-Flash-Next NVFP4 AgentX on B300 via SGLang with native NEXTN MTP. # Day-zero recipe; mirrors the B200 arm. TP1 per the cookbook's verified # single-node command: 126 GiB of NVFP4 weights fit on one B300. diff --git a/inferencex-e2e/infx/golden_al_distribution/README.md b/inferencex-e2e/infx/golden_al_distribution/README.md index fba2636b1e..48d632d954 100644 --- a/inferencex-e2e/infx/golden_al_distribution/README.md +++ b/inferencex-e2e/infx/golden_al_distribution/README.md @@ -24,6 +24,15 @@ AL is workload-dependent: a draft model's predictions are easier to accept in so Under the AgentX Guidelines, each model, thinking mode, and draft length has one committed golden AL. Once synthetic acceptance is enabled for a benchmark scenario, a submission may choose any supported draft length, but it may not substitute a different acceptance target. Different models keep their own SPEED-Bench-derived curves. All submissions evaluating the same model and mode use the same curve. +For Qwen3.5 srt-slurm launches, `apply_srt_recipe` uses the +`infx/srt_slurm/synthetic_acceptance.py` connector to select the committed curve +from the generation role's `speculative-num-steps` and `THINKING_MODE` after +caller overrides. The verification token is not an additional MTP step. For +SGLang MTP synthetic throughput, the connector rejects a generation role whose +`speculative-num-draft-tokens` is missing or is not an integer equal to +`speculative-num-steps + 1`. Manual acceptance-length settings are not needed. +Eval-only runs remove synthetic settings and use real verification. + vLLM supports this through synthetic rejection sampling. For example, an EAGLE3 run can inject the selected YAML value through `synthetic_acceptance_length`: ```bash diff --git a/inferencex-e2e/infx/golden_al_distribution/README_zh.md b/inferencex-e2e/infx/golden_al_distribution/README_zh.md index b7124fb0a4..68fb19d8d9 100644 --- a/inferencex-e2e/infx/golden_al_distribution/README_zh.md +++ b/inferencex-e2e/infx/golden_al_distribution/README_zh.md @@ -24,6 +24,8 @@ AL 取决于工作负载：草稿模型的预测在某些领域比其他领域 根据 AgentX 指南，每个模型、思考模式和草稿长度都有一个已提交的黄金 AL。当某个基准场景启用合成接受后，提交可以选择任意受支持的草稿长度，但不能替换为其他接受目标。不同模型保留各自基于 SPEED-Bench 测得的曲线。所有评估同一模型和模式的提交都使用同一条曲线。 +对于 Qwen3.5 的 srt-slurm 启动，`apply_srt_recipe` 通过 `infx/srt_slurm/synthetic_acceptance.py` 连接器，在应用调用方覆盖设置后，根据生成角色的 `speculative-num-steps` 和 `THINKING_MODE` 选择已提交的曲线。验证 token 不计为额外的 MTP 步数。对于采用合成接受的 SGLang MTP 吞吐测试，如果生成角色缺少 `speculative-num-draft-tokens`，或者其值不是等于 `speculative-num-steps + 1` 的整数，连接器会拒绝启动。无需手动设置接受长度。仅评测运行会移除合成设置并使用真实验证。 + vLLM 通过合成拒绝采样支持这一策略。例如，EAGLE3 运行可以通过 `synthetic_acceptance_length` 注入所选 YAML 值： ```bash diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 19ba42a60d..3ec2660a42 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -122,6 +122,19 @@ def build_overrides( al = golden_length( environment["MODEL_PREFIX"], spec, environment["THINKING_MODE"], golden_dir ) + if ( + engine == "sglang" + and environment["MODEL_PREFIX"] == "qwen3.5" + and spec.get("method") in ("eagle", "nextn", "mtp") + ): + # The generation role's MTP window includes one verification token. + drafts = generation.get("args", {}).get("speculative-num-draft-tokens") + expected = spec["num_speculative_tokens"] + 1 + if not isinstance(drafts, int) or isinstance(drafts, bool) or drafts != expected: + raise ValueError( + "Qwen3.5 SGLang generation role speculative-num-draft-tokens " + f"must be an integer equal to speculative-num-steps + 1 ({expected})" + ) overrides = [] variables = { "sglang": SGLANG_VARIABLES, diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index 6b43d9ac49..d22895001b 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -41,6 +41,7 @@ def golden_dir(tmp_path: Path) -> Path: directory.mkdir() for filename, model, acceptance in [ ("dsv4_mtp.yaml", "deepseek-v4-pro", 2.4), + ("qwen3.5_mtp.yaml", "qwen3.5", 2.6), ("dsv4-pro-0813-dspark.yaml", "deepseek-v4-pro-0813", 2.7), ("kimik3_dspark.yaml", "kimi-k3", 2.8), ( @@ -644,3 +645,76 @@ def test_sglang_conflicting_algorithm_aliases_fail(golden_dir: Path) -> None: } with pytest.raises(ValueError, match="Conflicting speculative-algorithm and speculative-algo"): build_overrides(recipe, "dynamo-sglang", ENV, golden_dir=golden_dir) + + +@pytest.mark.parametrize("drafts", [None, 3, True, 4.0]) +def test_qwen_mtp_rejects_invalid_generation_window(golden_dir: Path, drafts: Any) -> None: + args = {"speculative-algorithm": "NEXTN", "speculative-num-steps": 3} + if drafts is not None: + args["speculative-num-draft-tokens"] = drafts + with pytest.raises(ValueError, match=r"speculative-num-steps \+ 1 \(4\)"): + build_overrides( + {"roles": {"agg": {"args": args}}}, + "dynamo-sglang", + {**ENV, "MODEL_PREFIX": "qwen3.5"}, + golden_dir=golden_dir, + ) + + +def test_qwen_mtp_checks_effective_generation_window(tmp_path: Path, golden_dir: Path) -> None: + raw = { + "schema": 2, + "roles": { + "prefill": { + "args": {"speculative-algorithm": "NEXTN", "speculative-num-steps": 2} + }, + "decode": { + "args": { + "speculative-algorithm": "NEXTN", + "speculative-num-steps": 2, + "speculative-num-draft-tokens": 3, + } + }, + }, + } + recipe = tmp_path / "recipe.yaml" + recipe.write_text(yaml.safe_dump(raw)) + environment = {**ENV, "MODEL_PREFIX": "qwen3.5"} + arguments = ["--set", "roles.decode.args.speculative-num-steps=3"] + with pytest.raises(ValueError, match=r"speculative-num-steps \+ 1 \(4\)"): + plan_commands(str(recipe), "dynamo-sglang", arguments, environment, golden_dir=golden_dir) + commands = plan_commands( + str(recipe), + "dynamo-sglang", + [*arguments, "--set", "roles.decode.args.speculative-num-draft-tokens=4"], + environment, + golden_dir=golden_dir, + ) + result = apply_native(raw, commands[0]) + assert result["roles"]["decode"]["args"]["speculative-num-draft-tokens"] == 4 + assert result["roles"]["decode"]["env"]["SGLANG_SIMULATE_ACC_LEN"] == "2.6" + assert result["roles"]["prefill"]["args"]["speculative-num-steps"] == 2 + + +@pytest.mark.parametrize("environment", [{"EVAL_ONLY": "true"}, {"IS_AGENTIC": "0"}]) +def test_qwen_real_verification_does_not_require_mtp_window( + tmp_path: Path, environment: dict[str, str] +) -> None: + recipe = { + "roles": { + "agg": { + "args": {"speculative-algorithm": "NEXTN", "speculative-num-steps": 3}, + "env": {"SGLANG_SIMULATE_ACC_LEN": "9", "KEEP": "worker"}, + } + } + } + result = apply_native( + recipe, + build_overrides( + recipe, + "dynamo-sglang", + {**ENV, "MODEL_PREFIX": "qwen3.5", **environment}, + golden_dir=tmp_path / "absent", + ), + ) + assert result["roles"]["agg"]["env"] == {"KEEP": "worker"} diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 01f6d2d905..7e6d1b07d6 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9621,3 +9621,14 @@ - "Switch NIXL KV transfer to the C++ transfer-agent bounce buffer (max_average_descriptor_size 4MB, min_descriptor_count 1) and lower kv_cache_bounce_size_mb from 5120 to 1024." - "Move Dynamo on all six points to the ai-dynamo 1.6.0.dev20261010 wheel, which includes ai-dynamo/dynamo#15628 (routed node address selection for metadata and transport advertisement)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3707 + +- config-keys: + - qwen3.5-fp4-b300-dynamo-sglang-agentic-pareto + scenario-type: + - agentic-coding + description: + - "Add a Qwen3.5 NVFP4 B300 Dynamo-SGLang AgentX configuration with aggregate MTP recipes for TP2/EP2, TP2, and TP4 worker layouts." + - "Use the pinned lmsysorg/sglang:nightly-dev-20261013-7224fce1 image and validate each recipe's MTP step count against the committed Qwen3.5 thinking-on golden acceptance-length curve." + - "Serve the NextN draft layer at its shipped precision: no recipe sets SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE." + - "Replace aliases removed in this image (cuda-graph-max-bs becomes cuda-graph-max-bs-decode; SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1 becomes SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0) and drop settings that have no effect in it." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2978