From c01e7c4c5aab47921c68d2c9d937c27e720ed5f2 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Wed, 7 Oct 2026 18:46:55 -0700 Subject: [PATCH 1/2] feat(config): add Qwen3.8 GB300 AgentX recipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 Qwen3.8-Flash-Next NVFP4 的 GB300 AgentX 配置，使用原生 NEXTN MTP 和 SGLang HiCache。 --- .../sglang/gb300-fp4-mtp/agentic.yaml | 156 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 24 +++ inferencex-e2e/perf-changelog.yaml | 9 + 3 files changed, 189 insertions(+) create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..d92632b1a2 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml @@ -0,0 +1,156 @@ +# Stock ARM64 SGLang, one serving GPU per point. No source/package overlay. +# Size the host cache against the public GB300 profile's per-worker DRAM allowance. +# The matrix floors the allowance to 225 GB; H224 leaves pool-alignment headroom. +base: + schema: 2 + name: qwen3.8next-fp4-gb300-sglang-agentic-stock-v0521 + identity: + model: + repo: nvidia/Qwen3.8-Flash-Next-NVFP4 + revision: fc694b54fb0174e0913e6adf86691ef85a4ead47 + container: + image: lmsysorg/sglang:v0.5.21-cu130@sha256:1f176b3a1dd1205d2a462ab80ea9fc17769a32e53cb6f3c690efa2dc2c384977 + model: + path: hf:nvidia/Qwen3.8-Flash-Next-NVFP4 + container: lmsysorg/sglang:v0.5.21-cu130@sha256:1f176b3a1dd1205d2a462ab80ea9fc17769a32e53cb6f3c690efa2dc2c384977 + precision: fp4 + resources: + gpu_type: gb300 + gpus_per_node: 4 + slurm: + time_limit: '6:00:00' + health_check: + max_attempts: 1440 + interval_seconds: 5 + frontend: + type: sglang + enable_multiple_frontends: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 1 + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + NO_COLOR: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + # Explicitly disable the prohibited draft precision conversion path. + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '0' + args: + served-model-name: nvidia/Qwen3.8-Flash-Next-NVFP4 + revision: fc694b54fb0174e0913e6adf86691ef85a4ead47 + speculative-draft-model-revision: fc694b54fb0174e0913e6adf86691ef85a4ead47 + trust-remote-code: true + quantization: modelopt_mixed + context-length: 262144 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + attention-backend: trtllm_mha + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + linear-attn-verify-backend: triton + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer_lazy + mamba-track-interval: 256 + max-mamba-cache-size: 480 + mem-fraction-static: 0.88 + max-running-requests: 96 + page-size: 64 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + moe-runner-backend: flashinfer_trtllm + moe-a2a-backend: none + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: flashinfer_trtllm + # The native connector injects model-specific golden AL for throughput; + # quality evaluations use real verification. No manual AL override. + enable-linear-replayssm-spec: true + cuda-graph-backend-prefill: disabled + cuda-graph-backend-decode: full + cuda-graph-max-bs-decode: 96 + cuda-graph-bs-decode: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96] + kv-cache-dtype: bfloat16 + speculative-draft-kv-cache-dtype: bfloat16 + enable-hierarchical-cache: true + # One combined packed target/draft KV + Mamba host-pool budget. + # Verify actual allocations, not whole-node Slurm memory reservation. + hicache-size: 224 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_back + ple-offload-embedding: true + ple-offload-backend: pinned + enable-metrics: true + enable-cache-report: true + reasoning-parser: auto + tokenizer-worker-num: 1 + scheduler-recv-interval: 1 + decode-log-interval: 10 + stream-interval: 20 + watchdog-timeout: 1800 + random-seed: 35 + sbatch_directives: + mem: '0' + exclusive: '' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/Qwen3.8-Flash-Next-NVFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + # Native single-node binding verifies these against the generated point. + TOTAL_CPU_DRAM_GB: '225' + KV_OFFLOADING: dram + +# Only client concurrency changes; serving/admission/graph/cache settings stay fixed. +override_c1: + benchmark: + env: + CONC: '1' + +override_c2: + benchmark: + env: + CONC: '2' + +override_c4: + benchmark: + env: + CONC: '4' + +override_c8: + benchmark: + env: + CONC: '8' + +override_c16: + benchmark: + env: + CONC: '16' + +override_c32: + benchmark: + env: + CONC: '32' + +override_c48: + benchmark: + env: + CONC: '48' + +override_c64: + benchmark: + env: + CONC: '64' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index ac296d2809..ef8efc20bc 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9242,3 +9242,27 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark: - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml } + +# One serving GPU, stock ARM64 release, native AgentX client and proportional host cache. +qwen3.8next-fp4-gb300-sglang-agentic-mtp: + image: lmsysorg/sglang:v0.5.21-cu130@sha256:1f176b3a1dd1205d2a462ab80ea9fc17769a32e53cb6f3c690efa2dc2c384977 + model: nvidia/Qwen3.8-Flash-Next-NVFP4 + model-prefix: qwen3.8next + runner: cluster:gb300-nv + precision: fp4 + framework: sglang + multinode: false + scenarios: + agentic-coding: + # Use the GB300 profile's proportional per-worker host DRAM allowance. + # Generated allowance: 225 GB. Recipe H224 reserves alignment headroom. + - dram-utilization: 1.0 + search-space: + - tp: 1 + ep: 1 + dp-attn: false + spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [1, 2, 4, 8, 16, 32, 48, 64] + srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 77db75e1aa..d654ed494a 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9391,3 +9391,12 @@ - "Update the GB200 DeepSeek-V4.1-Flash vLLM AgentX image from nightly cd10ed6f to nightly-dev-arm64-cu130-ac9126e58aa7 and enable FlashInfer autotuning." - "Run TP4 at concurrency 1, 4, 8 and 32 and replace TP2 with DEP2 (TP1 x DP2 + EP2) at concurrency 8-32 and DEP4 (TP1 x DP4 + EP4) at concurrency 64-128, both with DeepGEMM MegaMoE and adaptive verification behind a consistent-hash vLLM Router. All points set gpu-memory-utilization 0.97." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3692 + +- config-keys: + - qwen3.8next-fp4-gb300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add Qwen3.8-Flash-Next NVFP4 GB300 TP1/EP1 AgentX with native NEXTN MTP and SGLang HiCache." + - "Pin the stock SGLang v0.5.21-cu130 image and checkpoint revision; enable engine metrics and cache reporting." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From e67ca6fc9ecf4cd57f27654bda8a3e9059923820 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Wed, 7 Oct 2026 18:47:14 -0700 Subject: [PATCH 2/2] fix(config): set Qwen3.8 GB300 changelog PR link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 更新 Qwen3.8 GB300 changelog 条目的 PR 链接。 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index d654ed494a..a71d02e50d 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9399,4 +9399,4 @@ description: - "Add Qwen3.8-Flash-Next NVFP4 GB300 TP1/EP1 AgentX with native NEXTN MTP and SGLang HiCache." - "Pin the stock SGLang v0.5.21-cu130 image and checkpoint revision; enable engine metrics and cache reporting." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3816