Repository navigation
ATOM SGLang Accuracy Validation #9
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ATOM SGLang Accuracy Validation | |
| on: | |
| schedule: | |
| # Nightly at 00:00 Beijing time (16:00 UTC on the previous day) | |
| - cron: '0 16 * * *' | |
| workflow_dispatch: | |
| # Manual runs use GitHub Actions' built-in branch dropdown ("Use workflow | |
| # from") as the ATOM branch selector, so users do not need to type branch | |
| # names. | |
| inputs: | |
| model_mi355: | |
| description: "Manual MI355 accuracy model selection" | |
| required: false | |
| type: choice | |
| default: none | |
| options: | |
| - none | |
| - all | |
| - DeepSeek-V4-Pro Prefix Cache TP8 | |
| - DeepSeek-V3.2-FP8 TP4 | |
| - DeepSeek-V3.2-FP8 TP4 DP4 EP4 | |
| - DeepSeek-V3.2-FP8 TP8 | |
| - DeepSeek-V3.2-FP8 TP8 DP8 EP8 | |
| - DeepSeek-R1-FP8 TP4 Online Quant | |
| - Kimi-K2.6-MXFP4 TP4 | |
| - Kimi-K2.6-MXFP4 TP8 | |
| - Kimi-K3 TP8 | |
| - MiniMax-M3-MXFP4 TP4 | |
| - MiniMax-M3-MXFP8 EAGLE3 TP4 MTP3 | |
| - Qwen3.8-2.4T-A95B-Quark-MXFP4 TP8 | |
| - Qwen3.8-Flash-Next-FP8 TP2 | |
| - Qwen3.5-35B-A3B-FP8 TP2 | |
| - Qwen3.5-35B-A3B TP2 | |
| - Qwen3.5-397B-A17B-FP8 TP4 | |
| - Qwen3.5-397B-A17B-FP8 TP8 | |
| - DeepSeek-R1-FP8 TP4 | |
| - DeepSeek-R1-FP8 TP8 | |
| - DeepSeek-R1-FP4 TP4 | |
| - DeepSeek-R1-FP4 TP4 DP4 EP4 | |
| - DeepSeek-R1-FP4 TP8 DP8 EP8 | |
| - DeepSeek-R1-FP4 TP8 | |
| - DeepSeek-R1-FP4 TP8 MTP3 | |
| - DeepSeek-R1-FP4 TP8 MTP1 | |
| - GLM-5.2-FP8 TP4 | |
| - GLM-5.2-FP8 TP4 PCP4 | |
| - GLM-5.2-MXFP4 TP4 | |
| model_mi308: | |
| description: "Manual MI308 accuracy model selection" | |
| required: false | |
| type: choice | |
| default: none | |
| options: | |
| - none | |
| - all | |
| - MI308 Qwen3.5-397B-A17B-FP8 TP4 | |
| - MI308 Qwen3.5-397B-A17B-FP8 TP8 | |
| - MI308 Qwen3.5-35B-A3B-FP8 TP1 | |
| - MI308 Qwen3.5-35B-A3B-PTPC-FP8 TP1 | |
| - MI308 Qwen3.8-Flash-Next-FP8 TP2 | |
| - MI308 DeepSeek-V4-Flash | |
| rebuild_atom_base_from_dockerfile: | |
| description: "Full build image before validation" | |
| required: false | |
| type: boolean | |
| default: false | |
| docker_image: | |
| description: "Optional Docker image override." | |
| required: false | |
| type: string | |
| default: "" | |
| upload_accuracy_to_dashboard: | |
| description: "Optional: upload SGLANG accuracy results to dashboard after this manual run" | |
| required: false | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| env: | |
| ATOM_BASE_NIGHTLY_IMAGE: rocm/atom-dev:latest | |
| NIGHTLY_SGLANG_IMAGE_TAG: rocm/atom-dev:sglang-latest | |
| VALIDATION_IMAGE_REPO: rocm/atom-dev | |
| GPU_ARCH: "gfx942;gfx950" | |
| AITER_REPO: "https://github.com/ROCm/aiter.git" | |
| AITER_COMMIT: "HEAD" | |
| jobs: | |
| # ---------- SGLANG full validation path (schedule / workflow_dispatch) ---------- | |
| prepare-sglang-image: | |
| name: Prepare SGLANG validation image | |
| runs-on: build-only-atom | |
| outputs: | |
| sglang_image_tag: ${{ steps.meta.outputs.sglang_image_tag }} | |
| cleanup_temp_image: ${{ steps.meta.outputs.cleanup_temp_image }} | |
| model_matrix: ${{ steps.meta.outputs.model_matrix }} | |
| model_matrix_mi355: ${{ steps.meta.outputs.model_matrix_mi355 }} | |
| model_matrix_mi308: ${{ steps.meta.outputs.model_matrix_mi308 }} | |
| has_model_cells_mi355: ${{ steps.meta.outputs.has_model_cells_mi355 }} | |
| has_model_cells_mi308: ${{ steps.meta.outputs.has_model_cells_mi308 }} | |
| steps: | |
| - name: Checkout selected ATOM branch | |
| uses: actions/checkout@v6 | |
| - name: Resolve SGLang version metadata | |
| run: python3 .github/scripts/resolve_sglang_metadata.py | |
| - name: Docker Login | |
| if: ${{ github.event_name == 'workflow_dispatch' && (github.ref_name != 'main' || inputs.rebuild_atom_base_from_dockerfile) }} | |
| run: | | |
| echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin | |
| - name: Resolve image source and model matrix | |
| id: meta | |
| env: | |
| DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} | |
| MODEL_MI355: ${{ inputs.model_mi355 || 'none' }} | |
| MODEL_MI308: ${{ inputs.model_mi308 || 'none' }} | |
| REBUILD_ATOM_BASE_FROM_DOCKERFILE: ${{ inputs.rebuild_atom_base_from_dockerfile }} | |
| INPUT_DOCKER_IMAGE: ${{ inputs.docker_image || '' }} | |
| run: | | |
| set -euo pipefail | |
| # Keep this Python model list as the single source of truth for SGLANG | |
| # full-validation coverage and manual dropdown filtering. | |
| python3 - <<'PY' >> "$GITHUB_OUTPUT" | |
| import json | |
| import os | |
| import sys | |
| event = os.environ["GITHUB_EVENT_NAME"] | |
| model_mi355 = os.environ.get("MODEL_MI355", "none") | |
| model_mi308 = os.environ.get("MODEL_MI308", "none") | |
| models = [ | |
| { | |
| "toggle_env": "RUN_DSV4_TP8", | |
| "model_name": "DeepSeek-V4-Pro TP8", | |
| "model_path": "deepseek-ai/DeepSeek-V4-Pro", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.9 --swa-full-tokens-ratio 0.1 --max-running-requests 256 --page-size 256 --disable-radix-cache --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 8, | |
| "accuracy_test_threshold": 0.94, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nAITER_BF16_FP8_MOE_BOUND=0\nATOM_MOE_GU_ITLV=1\nSGLANG_DEFAULT_THINKING=1\nSGLANG_DSV4_REASONING_EFFORT=max\nSGLANG_USE_AITER=1\nSGLANG_DSV4_FP4_EXPERTS=true\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSV4_MTP1_TP8", | |
| "model_name": "DeepSeek-V4-Pro MTP1 TP8", | |
| "model_path": "deepseek-ai/DeepSeek-V4-Pro", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.9 --swa-full-tokens-ratio 0.1 --max-running-requests 256 --page-size 256 --disable-radix-cache --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 8, | |
| "accuracy_test_threshold": 0.94, | |
| "mtp_accept_threshold": 0.80, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nAITER_BF16_FP8_MOE_BOUND=0\nATOM_MOE_GU_ITLV=1\nSGLANG_DEFAULT_THINKING=1\nSGLANG_DSV4_REASONING_EFFORT=max\nSGLANG_USE_AITER=1\nSGLANG_DSV4_FP4_EXPERTS=true\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nTORCHINDUCTOR_COMPILE_THREADS=128\nATOM_USE_TRITON_MOE=0\nATOM_SGLANG_V4_DISABLE_PRO_DRAFT_TRITON_MOE=1\nATOM_SGLANG_V4_ENABLE_DRAFT_EXTEND_CG=1", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSV4_MTP3_TP8", | |
| "model_name": "DeepSeek-V4-Pro MTP3 TP8", | |
| "model_path": "deepseek-ai/DeepSeek-V4-Pro", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.9 --swa-full-tokens-ratio 0.1 --max-running-requests 256 --page-size 256 --disable-radix-cache --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 8, | |
| "accuracy_test_threshold": 0.94, | |
| "mtp_accept_threshold": 0.55, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nAITER_BF16_FP8_MOE_BOUND=0\nATOM_MOE_GU_ITLV=1\nSGLANG_DEFAULT_THINKING=1\nSGLANG_DSV4_REASONING_EFFORT=max\nSGLANG_USE_AITER=1\nSGLANG_DSV4_FP4_EXPERTS=true\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nTORCHINDUCTOR_COMPILE_THREADS=128\nATOM_USE_TRITON_MOE=0\nATOM_SGLANG_V4_DISABLE_PRO_DRAFT_TRITON_MOE=1\nATOM_SGLANG_V4_ENABLE_DRAFT_EXTEND_CG=1", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSV4_PREFIX_CACHE_TP8", | |
| "model_name": "DeepSeek-V4-Pro Prefix Cache TP8", | |
| "model_path": "deepseek-ai/DeepSeek-V4-Pro", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --swa-full-tokens-ratio 0.1 --max-running-requests 256 --page-size 256 --enable-cache-report --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4", | |
| "accuracy_test_threshold": 0.94, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nAITER_BF16_FP8_MOE_BOUND=0\nATOM_MOE_GU_ITLV=1\nSGLANG_DEFAULT_THINKING=1\nSGLANG_DSV4_REASONING_EFFORT=max\nSGLANG_USE_AITER=1\nSGLANG_DSV4_FP4_EXPERTS=true\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_ENABLE_TORCH_COMPILE=1\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| # DeepSeek-R1 TP4 uses FP8 Q/KV decode so AITER dispatches its | |
| # native 32-head kernel on MI355X. | |
| { | |
| "toggle_env": "RUN_DSR1_FP8_TP4", | |
| "model_name": "DeepSeek-R1-FP8 TP4", | |
| "model_path": "deepseek-ai/DeepSeek-R1-0528", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 4 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.91, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_ENABLE_TORCH_COMPILE=1\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSV32_FP8_TP4", | |
| "model_name": "DeepSeek-V3.2-FP8 TP4", | |
| "model_path": "deepseek-ai/DeepSeek-V3.2", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 4 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.93, | |
| "env_vars": "SGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSV32_FP8_TP4_DP4_EP4", | |
| "model_name": "DeepSeek-V3.2-FP8 TP4 DP4 EP4", | |
| "model_path": "deepseek-ai/DeepSeek-V3.2", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 4 --data-parallel-size 4 --expert-parallel-size 4 --enable-dp-attention --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.93, | |
| "env_vars": "SGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nMORI_SHMEM_MODE=ISOLATION\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSV32_FP8_TP8_DP8_EP8", | |
| "model_name": "DeepSeek-V3.2-FP8 TP8 DP8 EP8", | |
| "model_path": "deepseek-ai/DeepSeek-V3.2", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --data-parallel-size 8 --expert-parallel-size 8 --enable-dp-attention --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.93, | |
| "env_vars": "SGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nMORI_SHMEM_MODE=ISOLATION\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSR1_FP8_TP4_ONLINE_QUANT", | |
| "model_name": "DeepSeek-R1-FP8 TP4 Online Quant", | |
| "model_path": "deepseek-ai/DeepSeek-R1-0528", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 4 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache --model-loader-extra-config '{\"online_quant_config\":{\"global_quant_config\":\"mxfp4\",\"exclude_layer\":[\"model.layers.*.self_attn.*\",\"model.layers.61.*\",\"lm_head\"]}}'", | |
| "accuracy_test_threshold": 0.91, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_ENABLE_TORCH_COMPILE=1\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_KIMI_K26_MXFP4_TP4", | |
| "model_name": "Kimi-K2.6-MXFP4 TP4", | |
| "model_path": "amd/Kimi-K2.6-MXFP4", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 4 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.8 --page-size 1 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.91, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nAITER_QUICK_REDUCE_QUANTIZATION=INT4\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_ENABLE_TORCH_COMPILE=1\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_KIMI_K26_MXFP4_TP8", | |
| "model_name": "Kimi-K2.6-MXFP4 TP8", | |
| "model_path": "amd/Kimi-K2.6-MXFP4", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.8 --page-size 1 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.91, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nAITER_QUICK_REDUCE_QUANTIZATION=INT4\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_ENABLE_TORCH_COMPILE=1\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_KIMI_K3_TP8", | |
| "model_name": "Kimi-K3 TP8", | |
| "model_path": "moonshotai/Kimi-K3", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --page-size 128 --context-length 16384 --mem-fraction-static 0.93 --disable-radix-cache", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 64, | |
| "accuracy_test_threshold": 0.94, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_PLUGINS=atom_sglang\nSGLANG_USE_AITER=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE=atom.plugin.sglang.models\nATOM_LOADER_NUM_THREADS=16\nATOM_SYNC_AFTER_LOAD=1\nATOM_DIST_TIMEOUT_SECONDS=3600\nATOM_USE_TRITON_GEMM=1\nAITER_USE_GROUPED_GEMM=0\nATOM_USE_TRITON_MOE=0\nAITER_FLYDSL_FORCE=1\nAITER_FORCE_GFX1250=0\nATOM_MLA_PAGE_SIZE=1\nATOM_USE_UNIFIED_ATTN=1\nATOM_FORCE_ATTN_TRITON=1", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_MINIMAX_M3_MXFP4_TP4", | |
| "model_name": "MiniMax-M3-MXFP4 TP4", | |
| "model_path": "amd/MiniMax-M3-MXFP4", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 4 --attention-backend aiter --page-size 128 --mem-fraction-static 0.8 --context-length 32768 --max-running-requests 128 --chunked-prefill-size 32768 --max-prefill-tokens 32768 --model-loader-extra-config '{\"online_quant_config\":{\"global_quant_config\":\"ptpc_fp8\",\"exclude_layer\":[\"lm_head\",\"model.embed_tokens\",\"vision_tower\",\"multi_modal_projector\",\"patch_merge_mlp\",\"*block_sparse_moe\"]}}' --json-model-override-args '{\"use_index_cache\":true,\"index_topk_freq\":4}' --kv-cache-dtype fp8_e4m3 --disable-radix-cache", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 64, | |
| "lm_eval_use_chat_completions": "1", | |
| "accuracy_test_threshold": 0.93, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nAITER_QUICK_REDUCE_QUANTIZATION=INT4\nSGLANG_USE_AITER=1\nATOM_FORCE_ATTN_TRITON=1\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_ENABLE_TORCH_COMPILE=1\nTORCHINDUCTOR_COMPILE_THREADS=128\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE=atom.plugin.sglang.models", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_MINIMAX_M3_MXFP8_EAGLE3_TP4_MTP3", | |
| "model_name": "MiniMax-M3-MXFP8 EAGLE3 TP4 MTP3", | |
| "model_path": "MiniMaxAI/MiniMax-M3-MXFP8", | |
| "extra_args": "--trust-remote-code --random-seed 0 --tensor-parallel-size 4 --attention-backend aiter --mem-fraction-static 0.75 --page-size 128 --context-length 32768 --max-running-requests 128 --model-loader-extra-config '{\"online_quant_config\":{\"global_quant_config\":\"ptpc_fp8\",\"exclude_layer\":[\"lm_head\",\"model.embed_tokens\",\"vision_tower\",\"multi_modal_projector\",\"patch_merge_mlp\",\"*block_sparse_moe\"]}}' --json-model-override-args '{\"use_index_cache\":true,\"index_topk_freq\":4}' --speculative-algorithm EAGLE3 --speculative-draft-model-path Inferact/MiniMax-M3-EAGLE3 --speculative-num-steps 3 --speculative-eagle-topk 1 --kv-cache-dtype fp8_e4m3 --disable-radix-cache --decode-log-interval 1", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 32, | |
| "lm_eval_use_chat_completions": "1", | |
| "lm_eval_extra_model_args": "max_gen_toks=16384", | |
| "accuracy_test_threshold": 0.93, | |
| "mtp_accept_threshold": 0.35, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE=atom.plugin.sglang.models\nAITER_QUICK_REDUCE_QUANTIZATION=INT4\nSGLANG_USE_AITER=1\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nATOM_FORCE_ATTN_TRITON=1\nSGLANG_ENABLE_TORCH_COMPILE=1\nTORCHINDUCTOR_COMPILE_THREADS=128\nATOM_SGLANG_EAGLE3_TP_VERIFY_BROADCAST=1", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_QWEN38_2_4T_A95B_QUARK_MXFP4_TP8", | |
| "model_name": "Qwen3.8-2.4T-A95B-Quark-MXFP4 TP8", | |
| "model_path": "amd/Qwen3.8-2.4T-A95B-Quark-MXFP4", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.9 --max-running-requests 256 --context-length 32768 --disable-radix-cache --page-size 16 --reasoning-parser qwen3 --enable-torch-compile --torch-compile-max-bs 64 --cuda-graph-backend-prefill tc_piecewise --cuda-graph-max-bs-prefill 16384 --cuda-graph-bs-prefill 8192 16384 --cuda-graph-tc-compiler inductor", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 64, | |
| "lm_eval_use_chat_completions": "1", | |
| "lm_eval_extra_model_args": "max_retries=3,timeout=1200,max_gen_toks=8192,tokenized_requests=False,trust_remote_code=True", | |
| "lm_eval_batch_size": 1, | |
| "lm_eval_gen_kwargs": "do_sample=True,temperature=1.0,top_p=0.95,top_k=20,min_tokens=1,reasoning_effort=low", | |
| "accuracy_test_threshold": 0.97, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_PLUGINS=atom_sglang\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE=atom.plugin.sglang.models\nSGLANG_USE_AITER=1\nAITER_LOG_LEVEL=WARNING\nATOM_AITER_FP8_PREFILL_ATTN=1\nATOM_SGLANG_PREFILL_COMPILE_ONLY=1\nTORCHINDUCTOR_COMPILE_THREADS=128\nSGLANG_MAMBA_SSM_DTYPE=bfloat16", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_QWEN38_FLASH_NEXT_FP8_TP2", | |
| "model_name": "Qwen3.8-Flash-Next-FP8 TP2", | |
| "model_path": "Qwen/Qwen3.8-Flash-Next-FP8", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 2 --ep-size 2 --attention-backend aiter --kv-cache-dtype bf16 --page-size 64 --mem-fraction-static 0.95 --context-length 16384 --max-running-requests 32 --disable-radix-cache --reasoning-parser qwen3 --cuda-graph-backend-decode full --cuda-graph-max-bs-decode 32", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 32, | |
| "lm_eval_use_chat_completions": "1", | |
| "lm_eval_extra_model_args": "max_retries=3,timeout=3600,tokenized_requests=False,trust_remote_code=True", | |
| "lm_eval_batch_size": 1, | |
| "lm_eval_gen_kwargs": "max_gen_toks=4096,until=<|im_end|>", | |
| "accuracy_test_threshold": 0.96, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_PLUGINS=atom_sglang\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE=atom.plugin.sglang.models\nSGLANG_USE_AITER=1\nSGLANG_AITER_KV_CACHE_LAYOUT=nhd\nAITER_LOG_LEVEL=WARNING\nAITER_MOE_PADDING_SIZE=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_QWEN35_35B_A3B_FP8_TP2", | |
| "model_name": "Qwen3.5-35B-A3B-FP8 TP2", | |
| "model_path": "Qwen/Qwen3.5-35B-A3B-FP8", | |
| "extra_args": "--tensor-parallel-size 2 --mem-fraction-static 0.9 --reasoning-parser qwen3 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.76, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION=0", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_QWEN35_35B_A3B_TP2", | |
| "model_name": "Qwen3.5-35B-A3B TP2", | |
| "model_path": "Qwen/Qwen3.5-35B-A3B", | |
| "extra_args": "--tensor-parallel-size 2 --mem-fraction-static 0.9 --reasoning-parser qwen3 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.83, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION=0", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_QWEN35_397B_A17B_FP8_TP4", | |
| "model_name": "Qwen3.5-397B-A17B-FP8 TP4", | |
| "model_path": "Qwen/Qwen3.5-397B-A17B-FP8", | |
| "extra_args": "--tensor-parallel-size 4 --mem-fraction-static 0.9 --reasoning-parser qwen3 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.83, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION=0", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_QWEN35_397B_A17B_FP8_TP8", | |
| "model_name": "Qwen3.5-397B-A17B-FP8 TP8", | |
| "model_path": "Qwen/Qwen3.5-397B-A17B-FP8", | |
| "extra_args": "--tensor-parallel-size 8 --mem-fraction-static 0.9 --reasoning-parser qwen3 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.83, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION=0", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSR1_FP8_TP8", | |
| "model_name": "DeepSeek-R1-FP8 TP8", | |
| "model_path": "deepseek-ai/DeepSeek-R1-0528", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.93, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_ENABLE_TORCH_COMPILE=1\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSR1_FP4_TP4", | |
| "model_name": "DeepSeek-R1-FP4 TP4", | |
| "model_path": "amd/DeepSeek-R1-0528-MXFP4-v2", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 4 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.91, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_ENABLE_TORCH_COMPILE=1\nTORCHINDUCTOR_COMPILE_THREADS=128\nATOM_LOADER_NUM_THREADS=1", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSR1_FP4_TP4_DP4_EP4", | |
| "model_name": "DeepSeek-R1-FP4 TP4 DP4 EP4", | |
| "model_path": "amd/DeepSeek-R1-0528-MXFP4-v2", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 4 --expert-parallel-size 4 --data-parallel-size 4 --enable-dp-attention --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.91, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nSGLANG_ENABLE_TORCH_COMPILE=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nMORI_SHMEM_MODE=ISOLATION\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSR1_FP4_TP8_DP8_EP8", | |
| "model_name": "DeepSeek-R1-FP4 TP8 DP8 EP8", | |
| "model_path": "amd/DeepSeek-R1-0528-MXFP4-v2", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --expert-parallel-size 8 --data-parallel-size 8 --enable-dp-attention --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.91, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nSGLANG_ENABLE_TORCH_COMPILE=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nMORI_SHMEM_MODE=ISOLATION\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSR1_FP4_TP8", | |
| "model_name": "DeepSeek-R1-FP4 TP8", | |
| "model_path": "amd/DeepSeek-R1-0528-MXFP4-v2", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.93, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_ENABLE_TORCH_COMPILE=1\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSR1_FP4_TP8_MTP3", | |
| "model_name": "DeepSeek-R1-FP4 TP8 MTP3", | |
| "model_path": "amd/DeepSeek-R1-0528-MXFP4-v2", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache --speculative-draft-model-path SGLang/DeepSeek-R1-NextN --speculative-algorithm NEXTN --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --max-running-requests 256 --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 160 192 224 256", | |
| "accuracy_test_threshold": 0.93, | |
| "mtp_accept_threshold": 0.50, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nSGLANG_ENABLE_SPEC_V2=1\nSGLANG_ENABLE_TORCH_COMPILE=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSR1_FP4_TP8_MTP1", | |
| "model_name": "DeepSeek-R1-FP4 TP8 MTP1", | |
| "model_path": "amd/DeepSeek-R1-0528-MXFP4-v2", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache --speculative-draft-model-path SGLang/DeepSeek-R1-NextN --speculative-algorithm NEXTN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 --max-running-requests 256 --cuda-graph-bs-decode 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 160 192 224 256", | |
| "accuracy_test_threshold": 0.93, | |
| "mtp_accept_threshold": 0.75, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nSGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nSGLANG_ENABLE_SPEC_V2=1\nSGLANG_ENABLE_TORCH_COMPILE=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_DSV32_FP8_TP8", | |
| "model_name": "DeepSeek-V3.2-FP8 TP8", | |
| "model_path": "deepseek-ai/DeepSeek-V3.2", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --attention-backend aiter --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --page-size 1 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.93, | |
| "env_vars": "SGLANG_AITER_FP8_PREFILL_ATTN=0\nSGLANG_USE_AITER=1\nATOM_ENABLE_DS_QKNORM_QUANT_FUSION=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_GLM52_FP8_TP4", | |
| "model_name": "GLM-5.2-FP8 TP4", | |
| "model_path": "zai-org/GLM-5.2-FP8", | |
| "extra_args": "--trust-remote-code --tp-size 4 --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.8 --disable-radix-cache --model-loader-extra-config '{\"online_quant_config\":{\"global_quant_config\":\"ptpc_fp8\",\"layer_quant_config\":{\"model.layers.*.mlp.experts\":\"per_block_fp8\"},\"exclude_layer\":[\"lm_head\",\"model.embed_tokens\",\"*.mlp.gate\"]}}'", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 65, | |
| "accuracy_test_threshold": 0.92, | |
| "env_vars": "AITER_QUICK_REDUCE_QUANTIZATION=INT4\nAITER_USE_FLYDSL_MOE_SORTING=1\nSGLANG_USE_AITER=1\nSGLANG_ENABLE_TORCH_COMPILE=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_GLM52_FP8_TP4_PCP4", | |
| "model_name": "GLM-5.2-FP8 TP4 PCP4", | |
| "model_path": "zai-org/GLM-5.2-FP8", | |
| "extra_args": "--trust-remote-code --tp-size 4 --enable-nsa-prefill-context-parallel --attn-cp-size 4 --nsa-prefill-cp-mode round-robin-split --max-prefill-tokens 30720 --chunked-prefill-size 30720 --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.85 --disable-radix-cache --model-loader-extra-config '{\"online_quant_config\":{\"global_quant_config\":\"ptpc_fp8\",\"layer_quant_config\":{\"model.layers.*.mlp.experts\":\"per_block_fp8\"},\"exclude_layer\":[\"lm_head\",\"model.embed_tokens\",\"*.mlp.gate\"]}}'", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 65, | |
| "accuracy_test_threshold": 0.92, | |
| "env_vars": "AITER_QUICK_REDUCE_QUANTIZATION=INT4\nAITER_USE_FLYDSL_MOE_SORTING=1\nSGLANG_USE_AITER=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nATOM_SGLANG_PCP_SIZE=4\nATOM_PCP_MOE_MERGE=1\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| { | |
| "toggle_env": "RUN_GLM52_MXFP4_TP4", | |
| "model_name": "GLM-5.2-MXFP4 TP4", | |
| "model_path": "amd/GLM-5.2-MXFP4", | |
| "extra_args": "--trust-remote-code --tp-size 4 --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.8 --disable-radix-cache --model-loader-extra-config '{\"online_quant_config\":{\"global_quant_config\":\"ptpc_fp8\",\"exclude_layer\":[\"lm_head\",\"model.embed_tokens\",\"*.mlp.gate\",\"*expert*\"]}}'", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 65, | |
| "accuracy_test_threshold": 0.92, | |
| "env_vars": "AITER_QUICK_REDUCE_QUANTIZATION=INT4\nAITER_USE_FLYDSL_MOE_SORTING=1\nSGLANG_USE_AITER=1\nSGLANG_ENABLE_TORCH_COMPILE=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-plugin-acc-validation-runner-sgl", | |
| }, | |
| ] | |
| # MI308 gsm8k accuracy: nightly schedule by default; manual runs select one case or all cases via model_mi308. | |
| mi308_models = [ | |
| { | |
| "model_name": "MI308 Qwen3.5-397B-A17B-FP8 TP4", | |
| "model_path": "Qwen/Qwen3.5-397B-A17B-FP8", | |
| "extra_args": "--tensor-parallel-size 4 --mem-fraction-static 0.9 --reasoning-parser qwen3 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.83, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION=0", | |
| "runner": "atom-mi308-8gpu-plugins-accuracy", | |
| }, | |
| { | |
| "model_name": "MI308 Qwen3.5-397B-A17B-FP8 TP8", | |
| "model_path": "Qwen/Qwen3.5-397B-A17B-FP8", | |
| "extra_args": "--tensor-parallel-size 8 --mem-fraction-static 0.9 --reasoning-parser qwen3 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.83, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION=0", | |
| "runner": "atom-mi308-8gpu-plugins-accuracy", | |
| }, | |
| { | |
| "model_name": "MI308 Qwen3.5-35B-A3B-FP8 TP1", | |
| "model_path": "Qwen/Qwen3.5-35B-A3B-FP8", | |
| "extra_args": "--tensor-parallel-size 1 --mem-fraction-static 0.9 --reasoning-parser qwen3 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.76, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION=0", | |
| "runner": "atom-mi308-8gpu-plugins-accuracy", | |
| }, | |
| { | |
| "model_name": "MI308 Qwen3.5-35B-A3B-PTPC-FP8 TP1", | |
| "model_path": "amd/Qwen3.5-35B-A3B-PTPC-FP8", | |
| "extra_args": "--tensor-parallel-size 1 --mem-fraction-static 0.9 --reasoning-parser qwen3 --disable-radix-cache", | |
| "accuracy_test_threshold": 0.76, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nATOM_ENABLE_QK_NORM_ROPE_CACHE_QUANT_FUSION=0", | |
| "runner": "atom-mi308-8gpu-plugins-accuracy", | |
| }, | |
| { | |
| "model_name": "MI308 Qwen3.8-Flash-Next-FP8 TP2", | |
| "model_path": "Qwen/Qwen3.8-Flash-Next-FP8", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 2 --ep-size 2 --attention-backend aiter --kv-cache-dtype bf16 --page-size 64 --mem-fraction-static 0.95 --context-length 16384 --max-running-requests 32 --disable-radix-cache --reasoning-parser qwen3 --cuda-graph-backend-decode full --cuda-graph-max-bs-decode 32", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 32, | |
| "lm_eval_use_chat_completions": "1", | |
| "lm_eval_extra_model_args": "max_retries=3,timeout=3600,tokenized_requests=False,trust_remote_code=True", | |
| "lm_eval_batch_size": 1, | |
| "lm_eval_gen_kwargs": "max_gen_toks=4096,until=<|im_end|>", | |
| "accuracy_test_threshold": 0.96, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nSGLANG_PLUGINS=atom_sglang\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nSGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE=atom.plugin.sglang.models\nSGLANG_USE_AITER=1\nSGLANG_AITER_KV_CACHE_LAYOUT=nhd\nAITER_LOG_LEVEL=WARNING\nAITER_MOE_PADDING_SIZE=128", | |
| "runner": "atom-mi308-8gpu-plugins-accuracy", | |
| }, | |
| { | |
| "model_name": "MI308 DeepSeek-V4-Flash", | |
| "model_path": "sgl-project/DeepSeek-V4-Flash-FP8", | |
| "extra_args": "--trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8_e4m3 --mem-fraction-static 0.9 --swa-full-tokens-ratio 0.1 --max-running-requests 256 --page-size 256 --disable-radix-cache --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4", | |
| "lm_eval_num_fewshot": 5, | |
| "lm_eval_num_concurrent": 8, | |
| "accuracy_test_threshold": 0.94, | |
| "env_vars": "SGLANG_DEFAULT_SERVER_ARGS=\nAITER_BF16_FP8_MOE_BOUND=0\nATOM_MOE_GU_ITLV=1\nSGLANG_DEFAULT_THINKING=1\nSGLANG_DSV4_REASONING_EFFORT=max\nSGLANG_USE_AITER=1\nSGLANG_EXTERNAL_MODEL_PACKAGE=atom.plugin.sglang.models\nTORCHINDUCTOR_COMPILE_THREADS=128", | |
| "runner": "atom-mi308-8gpu-plugins-accuracy", | |
| }, | |
| ] | |
| models.extend(mi308_models) | |
| selected = [] | |
| for model in models: | |
| toggle = model.get("toggle_env") | |
| if toggle is None: | |
| # MI308-only rows: all cases on nightly schedule; manual runs select one case or all cases. | |
| enabled = ( | |
| event != "workflow_dispatch" | |
| or model_mi308 == "all" | |
| or model["model_name"] == model_mi308 | |
| ) | |
| else: | |
| enabled = ( | |
| event != "workflow_dispatch" | |
| or model_mi355 == "all" | |
| or model["model_name"] == model_mi355 | |
| ) | |
| if enabled: | |
| selected.append({k: v for k, v in model.items() if k != "toggle_env"}) | |
| if event == "workflow_dispatch" and not selected: | |
| print("No models selected for manual run.", file=sys.stderr) | |
| sys.exit(1) | |
| def is_mi308_row(row): | |
| return "mi308" in str(row.get("runner", "")) | |
| sel355 = [r for r in selected if not is_mi308_row(r)] | |
| sel308 = [r for r in selected if is_mi308_row(r)] | |
| sep = (",", ":") | |
| full = json.dumps({"include": selected}, separators=sep) | |
| j355 = json.dumps({"include": sel355}, separators=sep) | |
| j308 = json.dumps({"include": sel308}, separators=sep) | |
| print(f"model_matrix={full}") | |
| print(f"model_matrix_mi355={j355}") | |
| print(f"model_matrix_mi308={j308}") | |
| print("has_model_cells_mi355=" + ("true" if sel355 else "false")) | |
| print("has_model_cells_mi308=" + ("true" if sel308 else "false")) | |
| PY | |
| REBUILD_FROM_DOCKERFILE="${REBUILD_ATOM_BASE_FROM_DOCKERFILE:-false}" | |
| USER_PROVIDED_SGLANG_IMAGE="${INPUT_DOCKER_IMAGE:-}" | |
| if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then | |
| if [ -n "${USER_PROVIDED_SGLANG_IMAGE}" ]; then | |
| echo "Manual run will use user-provided SGLANG image: ${USER_PROVIDED_SGLANG_IMAGE}" | |
| elif [ "${REBUILD_FROM_DOCKERFILE}" = "true" ]; then | |
| echo "Manual run selected full ATOM base rebuild from docker/atom_release.dockerfile" | |
| elif [ "${GITHUB_REF_NAME}" = "main" ]; then | |
| echo "Manual run on main reuses published SGLANG image" | |
| else | |
| echo "Manual run selected fast overlay rebuild from ${ATOM_BASE_NIGHTLY_IMAGE}" | |
| fi | |
| fi | |
| if [ -n "${USER_PROVIDED_SGLANG_IMAGE}" ]; then | |
| echo "sglang_image_tag=${USER_PROVIDED_SGLANG_IMAGE}" >> "$GITHUB_OUTPUT" | |
| echo "cleanup_temp_image=false" >> "$GITHUB_OUTPUT" | |
| echo "manual_base_image_tag=" >> "$GITHUB_OUTPUT" | |
| echo "rebuilt_atom_base_image_tag=" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| if [ "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ] || { [ "${GITHUB_REF_NAME}" = "main" ] && [ "${REBUILD_FROM_DOCKERFILE}" != "true" ]; }; then | |
| if RESOLUTION_JSON="$( | |
| python3 .github/scripts/resolve_atom_image.py \ | |
| --repository "${VALIDATION_IMAGE_REPO}" \ | |
| --reference-tag sglang-latest \ | |
| --preferred-version "${SGLANG_REF#v}" | |
| )"; then | |
| RESOLVED_SGLANG_IMAGE="$( | |
| RESOLUTION_JSON="${RESOLUTION_JSON}" python3 - <<'PY' | |
| import json | |
| import os | |
| resolution = json.loads(os.environ["RESOLUTION_JSON"]) | |
| print(resolution["resolved_image"]) | |
| PY | |
| )" | |
| else | |
| echo "::warning::Failed to resolve ${NIGHTLY_SGLANG_IMAGE_TAG} to an immutable reference; falling back to the floating tag." | |
| RESOLVED_SGLANG_IMAGE="${NIGHTLY_SGLANG_IMAGE_TAG}" | |
| fi | |
| if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then | |
| echo "Manual run on main reuses published SGLANG image: ${RESOLVED_SGLANG_IMAGE}" | |
| else | |
| echo "Scheduled run uses published SGLANG image: ${RESOLVED_SGLANG_IMAGE}" | |
| fi | |
| echo "sglang_image_tag=${RESOLVED_SGLANG_IMAGE}" >> "$GITHUB_OUTPUT" | |
| echo "cleanup_temp_image=false" >> "$GITHUB_OUTPUT" | |
| echo "manual_base_image_tag=" >> "$GITHUB_OUTPUT" | |
| echo "rebuilt_atom_base_image_tag=" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| SHORT_SHA="$(git rev-parse --short=12 HEAD)" | |
| MANUAL_BASE_IMAGE_TAG="atom_sglang_manual_base:${GITHUB_RUN_ID}-${SHORT_SHA}" | |
| REBUILT_ATOM_BASE_IMAGE_TAG="atom_sglang_rebuilt_base:${GITHUB_RUN_ID}-${SHORT_SHA}" | |
| SGLANG_IMAGE_TAG="${VALIDATION_IMAGE_REPO}:sglang-full-manual-${GITHUB_RUN_ID}-${SHORT_SHA}" | |
| BUILD_CONTEXT="$(mktemp -d)" | |
| trap 'rm -rf "${BUILD_CONTEXT}"' EXIT | |
| echo "sglang_image_tag=${SGLANG_IMAGE_TAG}" >> "$GITHUB_OUTPUT" | |
| echo "cleanup_temp_image=true" >> "$GITHUB_OUTPUT" | |
| if [ "${REBUILD_FROM_DOCKERFILE}" = "true" ]; then | |
| echo "manual_base_image_tag=" >> "$GITHUB_OUTPUT" | |
| echo "rebuilt_atom_base_image_tag=${REBUILT_ATOM_BASE_IMAGE_TAG}" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "manual_base_image_tag=${MANUAL_BASE_IMAGE_TAG}" >> "$GITHUB_OUTPUT" | |
| echo "rebuilt_atom_base_image_tag=" >> "$GITHUB_OUTPUT" | |
| fi | |
| git archive --format=tar HEAD | tar -xf - -C "${BUILD_CONTEXT}" | |
| if [ "${REBUILD_FROM_DOCKERFILE}" = "true" ]; then | |
| echo "Manual run rebuilds ATOM base and SGLANG image from branch ${GITHUB_REF_NAME}" | |
| DOCKER_BUILDKIT=1 docker build --pull --network=host --no-cache \ | |
| -t "${REBUILT_ATOM_BASE_IMAGE_TAG}" \ | |
| --build-arg GPU_ARCH="${GPU_ARCH}" \ | |
| --build-arg AITER_REPO="${AITER_REPO}" \ | |
| --build-arg AITER_COMMIT="${AITER_COMMIT}" \ | |
| --build-arg MAX_JOBS=64 \ | |
| --build-arg ATOM_REPO="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" \ | |
| --build-arg ATOM_COMMIT="${GITHUB_SHA}" \ | |
| -f "${BUILD_CONTEXT}/docker/atom_release.dockerfile" \ | |
| "${BUILD_CONTEXT}" | |
| DOCKER_BUILDKIT=1 docker build --network=host --no-cache \ | |
| -t "${SGLANG_IMAGE_TAG}" \ | |
| --build-arg SGLANG_BASE_IMAGE="${REBUILT_ATOM_BASE_IMAGE_TAG}" \ | |
| --build-arg MAX_JOBS=64 \ | |
| --build-arg SGLANG_REF="${SGLANG_REF}" \ | |
| --build-arg INSTALL_LM_EVAL=1 \ | |
| --build-arg INSTALL_FASTSAFETENSORS=1 \ | |
| -f "${BUILD_CONTEXT}/docker/sglang_release.dockerfile" \ | |
| "${BUILD_CONTEXT}" | |
| .github/scripts/docker_push_retry.sh "${SGLANG_IMAGE_TAG}" | |
| exit 0 | |
| fi | |
| cat <<'EOF' > "${BUILD_CONTEXT}/Dockerfile.mod" | |
| ARG ATOM_BASE_NIGHTLY_IMAGE | |
| FROM ${ATOM_BASE_NIGHTLY_IMAGE} | |
| ARG GPU_ARCH | |
| ARG AITER_REPO | |
| ARG AITER_COMMIT | |
| RUN pip install -U lm-eval[api] | |
| RUN pip show lm-eval || true | |
| RUN pip install hf_transfer | |
| RUN pip show hf_transfer || true | |
| RUN echo "=== Aiter version BEFORE uninstall ===" && pip show amd-aiter || true | |
| RUN pip uninstall -y amd-aiter | |
| RUN pip install --upgrade "pybind11>=3.0.1" | |
| RUN pip show pybind11 | |
| RUN rm -rf /app/aiter-test | |
| RUN git clone --depth 1 "${AITER_REPO}" /app/aiter-test && \ | |
| cd /app/aiter-test && \ | |
| git checkout "${AITER_COMMIT}" && \ | |
| git submodule sync && git submodule update --init --recursive && \ | |
| MAX_JOBS=64 PREBUILD_KERNELS=0 GPU_ARCHS="${GPU_ARCH}" python3 setup.py develop | |
| RUN echo "=== Aiter version AFTER installation ===" && pip show amd-aiter || true | |
| RUN echo "=== ATOM version BEFORE uninstall ===" && pip show atom || true | |
| RUN pip uninstall -y atom | |
| RUN rm -rf /app/ATOM | |
| COPY . /app/ATOM | |
| RUN cd /app/ATOM && pip install -e . | |
| RUN echo "=== ATOM version AFTER installation ===" && pip show atom || true | |
| EOF | |
| echo "Building manual SGLANG image from branch ${GITHUB_REF_NAME}" | |
| docker build --pull --network=host --no-cache \ | |
| -t "${MANUAL_BASE_IMAGE_TAG}" \ | |
| --build-arg ATOM_BASE_NIGHTLY_IMAGE="${ATOM_BASE_NIGHTLY_IMAGE}" \ | |
| --build-arg GPU_ARCH="${GPU_ARCH}" \ | |
| --build-arg AITER_REPO="${AITER_REPO}" \ | |
| --build-arg AITER_COMMIT="${AITER_COMMIT}" \ | |
| -f "${BUILD_CONTEXT}/Dockerfile.mod" \ | |
| "${BUILD_CONTEXT}" | |
| docker build --network=host --no-cache \ | |
| -t "${SGLANG_IMAGE_TAG}" \ | |
| --build-arg SGLANG_BASE_IMAGE="${MANUAL_BASE_IMAGE_TAG}" \ | |
| --build-arg MAX_JOBS=64 \ | |
| --build-arg SGLANG_REF="${SGLANG_REF}" \ | |
| --build-arg INSTALL_LM_EVAL=1 \ | |
| --build-arg INSTALL_FASTSAFETENSORS=1 \ | |
| -f "${BUILD_CONTEXT}/docker/sglang_release.dockerfile" \ | |
| "${BUILD_CONTEXT}" | |
| .github/scripts/docker_push_retry.sh "${SGLANG_IMAGE_TAG}" | |
| - name: Clean up builder images | |
| if: ${{ always() && steps.meta.outputs.cleanup_temp_image == 'true' }} | |
| run: | | |
| for image in \ | |
| "${{ steps.meta.outputs.sglang_image_tag }}" \ | |
| "${{ steps.meta.outputs.manual_base_image_tag }}" \ | |
| "${{ steps.meta.outputs.rebuilt_atom_base_image_tag }}"; do | |
| if [ -n "${image}" ]; then | |
| docker rmi "${image}" || true | |
| fi | |
| done | |
| sglang-model-accuracy-mi355: | |
| name: SGLANG Model Accuracy (MI355 / non-MI308 runners) | |
| needs: [prepare-sglang-image] | |
| if: >- | |
| always() | |
| && needs.prepare-sglang-image.result == 'success' | |
| && needs.prepare-sglang-image.outputs.has_model_cells_mi355 == 'true' | |
| permissions: | |
| actions: read | |
| contents: write | |
| uses: ./.github/workflows/atom-sglang-accuracy-validation-gpu-shard.yaml | |
| secrets: inherit | |
| with: | |
| shard_suffix: mi355 | |
| model_matrix_json: ${{ needs.prepare-sglang-image.outputs.model_matrix_mi355 }} | |
| sglang_image_tag: ${{ needs.prepare-sglang-image.outputs.sglang_image_tag }} | |
| upload_accuracy_artifact: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.upload_accuracy_to_dashboard) }} | |
| sglang-model-accuracy-mi308: | |
| name: SGLANG Model Accuracy (MI308 runners) | |
| needs: [prepare-sglang-image] | |
| if: >- | |
| always() | |
| && needs.prepare-sglang-image.result == 'success' | |
| && needs.prepare-sglang-image.outputs.has_model_cells_mi308 == 'true' | |
| permissions: | |
| actions: read | |
| contents: write | |
| uses: ./.github/workflows/atom-sglang-accuracy-validation-gpu-shard.yaml | |
| secrets: inherit | |
| with: | |
| shard_suffix: mi308 | |
| model_matrix_json: ${{ needs.prepare-sglang-image.outputs.model_matrix_mi308 }} | |
| sglang_image_tag: ${{ needs.prepare-sglang-image.outputs.sglang_image_tag }} | |
| upload_accuracy_artifact: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.upload_accuracy_to_dashboard) }} | |
| accuracy-dashboard: | |
| name: Update SGLANG accuracy dashboard | |
| needs: [sglang-model-accuracy-mi355, sglang-model-accuracy-mi308] | |
| if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.upload_accuracy_to_dashboard)) }} | |
| concurrency: | |
| group: gh-pages-deploy | |
| cancel-in-progress: false | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: read | |
| contents: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Prepare accuracy artifact directory | |
| run: mkdir -p /tmp/accuracy-results | |
| - name: Download accuracy artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| path: /tmp/accuracy-results | |
| pattern: accuracy-* | |
| continue-on-error: true | |
| - name: Count downloaded accuracy artifacts | |
| id: downloaded | |
| run: | | |
| json_count=$(find /tmp/accuracy-results -type f -name '*.json' | wc -l | tr -d ' ') | |
| echo "json_count=${json_count}" >> "$GITHUB_OUTPUT" | |
| echo "Downloaded ${json_count} SGLANG accuracy JSON files" | |
| - name: List downloaded accuracy artifacts | |
| if: steps.downloaded.outputs.json_count != '0' | |
| run: | | |
| echo "=== Downloaded SGLANG accuracy artifacts ===" | |
| find /tmp/accuracy-results -type f -name '*.json' | head -20 || echo "No JSON files found" | |
| - name: Transform SGLANG accuracy results for dashboard | |
| id: transform | |
| if: steps.downloaded.outputs.json_count != '0' | |
| run: | | |
| python3 .github/scripts/accuracy_to_dashboard.py \ | |
| /tmp/accuracy-results \ | |
| --output accuracy-benchmark-input.json \ | |
| --models .github/benchmark/sglang_models_accuracy.json \ | |
| --backend "ATOM-SGLang" \ | |
| --run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| echo "=== Generated entries ===" | |
| cat accuracy-benchmark-input.json | |
| entries_count=$(python3 -c "import json; data=json.load(open('accuracy-benchmark-input.json')); print(len(data))") | |
| echo "entries_count=${entries_count}" >> "$GITHUB_OUTPUT" | |
| if [ "${entries_count}" = "0" ]; then | |
| echo "::warning::No valid SGLANG accuracy entries were produced. Check that the result JSON files contain 'results.gsm8k[\"exact_match,flexible-extract\"]'." | |
| fi | |
| - name: Prepare existing dashboard data | |
| if: steps.transform.outputs.entries_count != '0' && steps.transform.outputs.entries_count != '' | |
| id: prepare_dashboard_data | |
| uses: ./.github/actions/prepare-dashboard-data | |
| with: | |
| data-path: benchmark-dashboard/data.js | |
| - name: Store SGLANG accuracy result to dashboard | |
| if: steps.transform.outputs.entries_count != '0' && steps.transform.outputs.entries_count != '' | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| tool: customBiggerIsBetter | |
| output-file-path: accuracy-benchmark-input.json | |
| gh-pages-branch: gh-pages | |
| benchmark-data-dir-path: benchmark-dashboard | |
| auto-push: false | |
| max-items-in-chart: 300 | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Validate dashboard data after write | |
| if: steps.transform.outputs.entries_count != '0' && steps.transform.outputs.entries_count != '' | |
| uses: ./.github/actions/validate-dashboard-data | |
| with: | |
| data-path: benchmark-dashboard/data.js | |
| min-entries: ${{ steps.prepare_dashboard_data.outputs.entry_count }} | |
| - name: Push dashboard data | |
| if: steps.transform.outputs.entries_count != '0' && steps.transform.outputs.entries_count != '' | |
| run: git push origin gh-pages | |