diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..d92632b1a2 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml @@ -0,0 +1,156 @@ +# Stock ARM64 SGLang, one serving GPU per point. No source/package overlay. +# Size the host cache against the public GB300 profile's per-worker DRAM allowance. +# The matrix floors the allowance to 225 GB; H224 leaves pool-alignment headroom. +base: + schema: 2 + name: qwen3.8next-fp4-gb300-sglang-agentic-stock-v0521 + identity: + model: + repo: nvidia/Qwen3.8-Flash-Next-NVFP4 + revision: fc694b54fb0174e0913e6adf86691ef85a4ead47 + container: + image: lmsysorg/sglang:v0.5.21-cu130@sha256:1f176b3a1dd1205d2a462ab80ea9fc17769a32e53cb6f3c690efa2dc2c384977 + model: + path: hf:nvidia/Qwen3.8-Flash-Next-NVFP4 + container: lmsysorg/sglang:v0.5.21-cu130@sha256:1f176b3a1dd1205d2a462ab80ea9fc17769a32e53cb6f3c690efa2dc2c384977 + precision: fp4 + resources: + gpu_type: gb300 + gpus_per_node: 4 + slurm: + time_limit: '6:00:00' + health_check: + max_attempts: 1440 + interval_seconds: 5 + frontend: + type: sglang + enable_multiple_frontends: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 1 + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + NO_COLOR: '1' + SGLANG_TIMEOUT_KEEP_ALIVE: '1800' + # Explicitly disable the prohibited draft precision conversion path. + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '0' + args: + served-model-name: nvidia/Qwen3.8-Flash-Next-NVFP4 + revision: fc694b54fb0174e0913e6adf86691ef85a4ead47 + speculative-draft-model-revision: fc694b54fb0174e0913e6adf86691ef85a4ead47 + trust-remote-code: true + quantization: modelopt_mixed + context-length: 262144 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 1 + expert-parallel-size: 1 + attention-backend: trtllm_mha + linear-attn-prefill-backend: flashinfer + linear-attn-decode-backend: flashinfer + linear-attn-verify-backend: triton + mamba-ssm-dtype: bfloat16 + mamba-radix-cache-strategy: extra_buffer_lazy + mamba-track-interval: 256 + max-mamba-cache-size: 480 + mem-fraction-static: 0.88 + max-running-requests: 96 + page-size: 64 + chunked-prefill-size: 16384 + max-prefill-tokens: 16384 + moe-runner-backend: flashinfer_trtllm + moe-a2a-backend: none + disable-shared-experts-fusion: true + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + speculative-moe-runner-backend: flashinfer_trtllm + # The native connector injects model-specific golden AL for throughput; + # quality evaluations use real verification. No manual AL override. + enable-linear-replayssm-spec: true + cuda-graph-backend-prefill: disabled + cuda-graph-backend-decode: full + cuda-graph-max-bs-decode: 96 + cuda-graph-bs-decode: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96] + kv-cache-dtype: bfloat16 + speculative-draft-kv-cache-dtype: bfloat16 + enable-hierarchical-cache: true + # One combined packed target/draft KV + Mamba host-pool budget. + # Verify actual allocations, not whole-node Slurm memory reservation. + hicache-size: 224 + hicache-io-backend: kernel + hicache-mem-layout: page_first + hicache-write-policy: write_back + ple-offload-embedding: true + ple-offload-backend: pinned + enable-metrics: true + enable-cache-report: true + reasoning-parser: auto + tokenizer-worker-num: 1 + scheduler-recv-interval: 1 + decode-log-interval: 10 + stream-interval: 20 + watchdog-timeout: 1800 + random-seed: 35 + sbatch_directives: + mem: '0' + exclusive: '' + srun_options: + mem: '0' + container-remap-root: '' + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: nvidia/Qwen3.8-Flash-Next-NVFP4 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + # Native single-node binding verifies these against the generated point. + TOTAL_CPU_DRAM_GB: '225' + KV_OFFLOADING: dram + +# Only client concurrency changes; serving/admission/graph/cache settings stay fixed. +override_c1: + benchmark: + env: + CONC: '1' + +override_c2: + benchmark: + env: + CONC: '2' + +override_c4: + benchmark: + env: + CONC: '4' + +override_c8: + benchmark: + env: + CONC: '8' + +override_c16: + benchmark: + env: + CONC: '16' + +override_c32: + benchmark: + env: + CONC: '32' + +override_c48: + benchmark: + env: + CONC: '48' + +override_c64: + benchmark: + env: + CONC: '64' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index d8eaafffb9..fc323dd917 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9374,3 +9374,27 @@ dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg: tp: 2 ep: 2 dp-attn: false + +# One serving GPU, stock ARM64 release, native AgentX client and proportional host cache. +qwen3.8next-fp4-gb300-sglang-agentic-mtp: + image: lmsysorg/sglang:v0.5.21-cu130@sha256:1f176b3a1dd1205d2a462ab80ea9fc17769a32e53cb6f3c690efa2dc2c384977 + model: nvidia/Qwen3.8-Flash-Next-NVFP4 + model-prefix: qwen3.8next + runner: cluster:gb300-nv + precision: fp4 + framework: sglang + multinode: false + scenarios: + agentic-coding: + # Use the GB300 profile's proportional per-worker host DRAM allowance. + # Generated allowance: 225 GB. Recipe H224 reserves alignment headroom. + - dram-utilization: 1.0 + search-space: + - tp: 1 + ep: 1 + dp-attn: false + spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [1, 2, 4, 8, 16, 32, 48, 64] + srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index bb30c845c3..89f9810d2e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9488,3 +9488,12 @@ - "No data-type or precision change to the EAGLE3 draft model (Inferact/MiniMax-M3-EAGLE3-GQA): online_quant_config (ptpc_fp8), kv_cache_dtype fp8 and index-cache-dtype fp8 are unchanged; mono decode changes the target model's decode kernels, not draft precision." - "更新镜像至 rocm/atom-dev:nightly_202610070333(ATOM 0.1.7.dev60+g7fc0926c6,含 ROCm/ATOM#2483 的 mono step fence 修复);去掉 TP4 C15/C20/C24/C28 的 ATOM_MONO_ENABLE=0,所有 TP4 点位启用 MiniMax-M3 mono decode(每步最多 16 个 token,即 EAGLE3 K3 下 4 个请求,覆盖 C1/C2/C4);ATOM 只为 TP4 构建 mono,TP2 走非融合路径。新增 TP2 C4,LMCache 版本记录为 0.5.6.dev139+gf22dec28。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3750 + +- config-keys: + - qwen3.8next-fp4-gb300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add Qwen3.8-Flash-Next NVFP4 GB300 TP1/EP1 AgentX with native NEXTN MTP and SGLang HiCache." + - "Pin the stock SGLang v0.5.21-cu130 image and checkpoint revision; enable engine metrics and cache reporting." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3816