model_configs.json

{
  "dense_models": {
    "Llama-2-7B": {
      "source": "https://huggingface.co/meta-llama/Llama-2-7b/blob/main/config.json",
      "seqlen": 4096,
      "hidden_size": 4096,
      "intermediate_size": 11008,
      "num_attention_heads": 32,
      "num_key_value_heads": 32,
      "head_dim": 128
    },
    "Llama-2-70B": {
      "source": "https://huggingface.co/meta-llama/Llama-2-70b/blob/main/config.json",
      "seqlen": 4096,
      "hidden_size": 8192,
      "intermediate_size": 28672,
      "num_attention_heads": 64,
      "num_key_value_heads": 8,
      "head_dim": 128
    },
    "Llama-3.1-8B": {
      "source": "https://huggingface.co/meta-llama/Llama-3.1-8B/blob/main/config.json",
      "seqlen": 8192,
      "hidden_size": 4096,
      "intermediate_size": 14336,
      "num_attention_heads": 32,
      "num_key_value_heads": 8,
      "head_dim": 128
    },
    "Llama-3.1-405B": {
      "source": "https://huggingface.co/meta-llama/Llama-3.1-405B/blob/main/config.json",
      "seqlen": 8192,
      "hidden_size": 16384,
      "intermediate_size": 53248,
      "num_attention_heads": 128,
      "num_key_value_heads": 8,
      "head_dim": 128
    },
    "Qwen2.5-7B": {
      "source": "https://modelscope.cn/models/Qwen/Qwen2.5-7B-Instruct/file/view/master/config.json",
      "seqlen": 8192,
      "hidden_size": 3584,
      "intermediate_size": 18944,
      "num_attention_heads": 28,
      "num_key_value_heads": 4,
      "head_dim": 128
    },
    "Qwen2.5-72B": {
      "source": "https://modelscope.cn/models/Qwen/Qwen2.5-72B-Instruct",
      "seqlen": 8192,
      "hidden_size": 8192,
      "intermediate_size": 29568,
      "num_attention_heads": 64,
      "num_key_value_heads": 8,
      "head_dim": 128
    },
    "Mistral-7B": {
      "source": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1/blob/main/config.json",
      "seqlen": 4096,
      "hidden_size": 4096,
      "intermediate_size": 14336,
      "num_attention_heads": 32,
      "num_key_value_heads": 8,
      "head_dim": 128
    }
  },
  "moe_models": {
    "DeepSeek-V3": {
      "source": "https://modelscope.cn/models/deepseek-ai/DeepSeek-V3/file/view/master/config.json",
      "attention_type": "MLA",
      "n_routed_experts": 256,
      "moe_intermediate_size": 2048,
      "hidden_size": 7168,
      "num_attention_heads": 128,
      "num_key_value_heads": 128,
      "head_dim_qk": 192,
      "head_dim_v": 128,
      "seqlen": 4096,
      "num_experts": 256,
      "num_topk": 8
    },
    "DeepSeek-V2": {
      "source": "https://modelscope.cn/models/deepseek-ai/DeepSeek-V2/file/view/master/config.json",
      "attention_type": "MLA",
      "n_routed_experts": 160,
      "moe_intermediate_size": 1536,
      "hidden_size": 5120,
      "num_attention_heads": 128,
      "num_key_value_heads": 128,
      "head_dim_qk": 192,
      "head_dim_v": 128,
      "seqlen": 4096,
      "num_experts": 160,
      "num_topk": 6
    },
    "DeepSeek-V2-Lite": {
      "source": "https://modelscope.cn/models/deepseek-ai/DeepSeek-V2-Lite/file/view/master/config.json",
      "attention_type": "MLA",
      "n_routed_experts": 64,
      "moe_intermediate_size": 1408,
      "hidden_size": 2048,
      "num_attention_heads": 16,
      "num_key_value_heads": 16,
      "head_dim_qk": 192,
      "head_dim_v": 128,
      "seqlen": 4096,
      "num_experts": 64,
      "num_topk": 6
    },
    "DeepSeek-V4-Flash": {
      "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/main/config.json",
      "attention_type": "HybridSparse",
      "n_routed_experts": 256,
      "n_shared_experts": 1,
      "moe_intermediate_size": 2048,
      "hidden_size": 4096,
      "num_attention_heads": 64,
      "num_key_value_heads": 1,
      "head_dim": 512,
      "head_dim_qk": 576,
      "head_dim_v": 512,
      "qk_rope_head_dim": 64,
      "q_lora_rank": 1024,
      "o_lora_rank": 1024,
      "o_groups": 8,
      "num_hidden_layers": 43,
      "num_hash_layers": 3,
      "sliding_window": 128,
      "index_n_heads": 64,
      "index_head_dim": 128,
      "index_topk": 512,
      "max_position_embeddings": 1048576,
      "seqlen": 4096,
      "num_experts": 256,
      "num_topk": 6
    },
    "DeepSeek-V4-Pro": {
      "source": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/blob/main/config.json",
      "attention_type": "HybridSparse",
      "n_routed_experts": 384,
      "n_shared_experts": 1,
      "moe_intermediate_size": 3072,
      "hidden_size": 7168,
      "num_attention_heads": 128,
      "num_key_value_heads": 1,
      "head_dim": 512,
      "head_dim_qk": 576,
      "head_dim_v": 512,
      "qk_rope_head_dim": 64,
      "q_lora_rank": 1536,
      "o_lora_rank": 1024,
      "o_groups": 16,
      "num_hidden_layers": 61,
      "num_hash_layers": 3,
      "sliding_window": 128,
      "index_n_heads": 64,
      "index_head_dim": 128,
      "index_topk": 1024,
      "max_position_embeddings": 1048576,
      "seqlen": 4096,
      "num_experts": 384,
      "num_topk": 6
    },
    "Grok-2": {
      "source": "https://huggingface.co/xai-org/grok-2/blob/main/config.json",
      "attention_type": "GQA",
      "n_routed_experts": 8,
      "moe_intermediate_size": 16384,
      "hidden_size": 8192,
      "num_attention_heads": 64,
      "num_key_value_heads": 8,
      "head_dim": 128,
      "seqlen": 8192,
      "num_experts": 8,
      "num_topk": 2
    },
    "Qwen3-30B-A3B": {
      "source": "https://modelscope.cn/models/Qwen/Qwen3-30B-A3B-Instruct-2507/file/view/master/config.json",
      "attention_type": "GQA",
      "n_routed_experts": 128,
      "moe_intermediate_size": 768,
      "hidden_size": 2048,
      "num_attention_heads": 32,
      "num_key_value_heads": 4,
      "head_dim": 128,
      "seqlen": 8192,
      "num_experts": 128,
      "num_topk": 8
    },
    "Qwen3-235B-A22B": {
      "source": "https://modelscope.cn/models/Qwen/Qwen3-235B-A22B-Instruct-2507",
      "attention_type": "GQA",
      "n_routed_experts": 128,
      "moe_intermediate_size": 1536,
      "hidden_size": 4096,
      "num_attention_heads": 64,
      "num_key_value_heads": 4,
      "head_dim": 128,
      "seqlen": 8192,
      "num_experts": 128,
      "num_topk": 8
    },
    "Mixtral-8x7B": {
      "source": "https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1/blob/main/config.json",
      "attention_type": "GQA",
      "n_routed_experts": 8,
      "moe_intermediate_size": 14336,
      "hidden_size": 4096,
      "num_attention_heads": 32,
      "num_key_value_heads": 8,
      "head_dim": 128,
      "seqlen": 4096,
      "num_experts": 8,
      "num_topk": 2
    },
    "Mixtral-8x22B": {
      "source": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1/blob/main/config.json",
      "attention_type": "GQA",
      "n_routed_experts": 8,
      "moe_intermediate_size": 16384,
      "hidden_size": 6144,
      "num_attention_heads": 48,
      "num_key_value_heads": 8,
      "head_dim": 128,
      "seqlen": 8192,
      "num_experts": 8,
      "num_topk": 2
    },
    "Kimi-K2": {
      "source": "https://huggingface.co/moonshotai/Kimi-K2-Instruct/blob/main/config.json",
      "attention_type": "MLA",
      "n_routed_experts": 384,
      "moe_intermediate_size": 2048,
      "hidden_size": 7168,
      "num_attention_heads": 64,
      "num_key_value_heads": 64,
      "head_dim_qk": 192,
      "head_dim_v": 128,
      "seqlen": 4096,
      "num_experts": 384,
      "num_topk": 8
    },
    "MoE-1T": {
      "source": "https://github.com/AMD-AGI/Primus/blob/main/primus/configs/models/megatron/moe_1T.yaml",
      "attention_type": "GQA",
      "n_routed_experts": 224,
      "moe_intermediate_size": 1920,
      "hidden_size": 8192,
      "num_attention_heads": 64,
      "num_key_value_heads": 8,
      "head_dim": 128,
      "seqlen": 4096,
      "num_experts": 224,
      "num_topk": 8
    }
  },
  "attention_models": {
    "Kimi-K2": {
      "source": "https://huggingface.co/moonshotai/Kimi-K2-Instruct/raw/main/config.json",
      "serve": {
        "cookbook": "https://github.com/sgl-project/sglang/blob/main/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.mdx",
        "tp": 8,
        "attention_backend": "auto",
        "kv_cache_dtype": "fp8_e4m3",
        "env": {"SGLANG_ROCM_FUSED_DECODE_MLA": "0"},
        "notes": "AMD: MUST set SGLANG_ROCM_FUSED_DECODE_MLA=0 before launch. Cookbook recommends --kv-cache-dtype fp8_e4m3 (-50% KV mem). EP/DP: --ep / --dp 4 --enable-dp-attention. ROCm verified on MI300X. Attention auto-selects triton (kv-heads=64 not in {16,128})."
      },
      "attention_type": "MLA",
      "architectures": "DeepseekV3ForCausalLM",
      "hidden_size": 7168,
      "num_attention_heads": 64,
      "num_key_value_heads": 64,
      "qk_nope_head_dim": 128,
      "qk_rope_head_dim": 64,
      "head_dim_qk": 192,
      "v_head_dim": 128,
      "kv_lora_rank": 512,
      "q_lora_rank": 1536,
      "rope_scaling": {"type": "yarn", "factor": 32.0, "mscale": 1.0, "mscale_all_dim": 1.0, "original_max_position_embeddings": 4096},
      "num_hidden_layers": 61,
      "max_position_embeddings": 131072
    },
    "DeepSeek-R1": {
      "source": "https://huggingface.co/deepseek-ai/DeepSeek-R1/raw/main/config.json",
      "serve": {
        "cookbook": "https://github.com/sgl-project/sglang/blob/main/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-R1.mdx",
        "tp": 8,
        "attention_backend": "auto",
        "kv_cache_dtype": "auto",
        "quantization": "FP4 (nvidia/DeepSeek-R1-0528-FP4-v2) or amd/DeepSeek-R1-MXFP4 recommended on MI355X",
        "env": {},
        "notes": "Official FP8 ckpt is already FP8: do NOT pass --quantization fp8. High-throughput: --enable-dp-attention --tp 8 --dp 8. Attention auto-selects aiter on HIP (MLA absorbed decode)."
      },
      "attention_type": "MLA",
      "architectures": "DeepseekV3ForCausalLM",
      "hidden_size": 7168,
      "num_attention_heads": 128,
      "num_key_value_heads": 128,
      "qk_nope_head_dim": 128,
      "qk_rope_head_dim": 64,
      "head_dim_qk": 192,
      "v_head_dim": 128,
      "kv_lora_rank": 512,
      "q_lora_rank": 1536,
      "rope_scaling": {"type": "yarn", "factor": 40, "mscale": 1.0, "mscale_all_dim": 1.0, "original_max_position_embeddings": 4096},
      "num_hidden_layers": 61,
      "max_position_embeddings": 163840
    },
    "MiniMax-M2": {
      "source": "https://huggingface.co/MiniMaxAI/MiniMax-M2/raw/main/config.json",
      "serve": {
        "cookbook": "https://github.com/sgl-project/sglang/blob/main/docs_new/cookbook/autoregressive/MiniMax/MiniMax-M2.mdx",
        "tp": 8,
        "attention_backend": "auto",
        "kv_cache_dtype": "auto",
        "mem_fraction_static": 0.85,
        "docker_image": "lmsysorg/sglang:v0.5.9-rocm720-mi30x",
        "env": {},
        "notes": "AMD docker mods: rename transformers.py -> hf_transformers_model.py; comment out @torch.compile in minimax_m2.py. GQA, attention auto-selects aiter. --mem-fraction-static 0.85."
      },
      "attention_type": "GQA",
      "architectures": "MiniMaxM2ForCausalLM",
      "attention_note": "full softmax attention on all 62 layers (attn_type_list all=1); use_qk_norm=true; NOT lightning/linear/hybrid (M2 abandoned MiniMax-Text-01 hybrid)",
      "hidden_size": 3072,
      "num_attention_heads": 48,
      "num_key_value_heads": 8,
      "head_dim": 128,
      "num_hidden_layers": 62,
      "max_position_embeddings": 196608
    },
    "GLM-5": {
      "source": "https://huggingface.co/zai-org/GLM-5/raw/main/config.json",
      "serve": {
        "cookbook": "https://github.com/sgl-project/sglang/blob/main/docs_new/cookbook/autoregressive/GLM/GLM-5.mdx",
        "tp": 8,
        "attention_backend": "dsa",
        "dsa_decode_backend": "tilelang",
        "dsa_prefill_backend": "tilelang",
        "kv_cache_dtype": "auto",
        "extra_flags": ["--chunked-prefill-size 131072", "--watchdog-timeout 1200", "--mem-fraction-static 0.80"],
        "env": {},
        "notes": "AMD: --dsa-prefill-backend tilelang --dsa-decode-backend tilelang (aiter is decode-only and has NO fp8 KV kernel). EAGLE spec-decode NOT supported on AMD. Optional IndexCache via --json-model-override-args index_topk_pattern. Same structure as DeepSeek-V3.2."
      },
      "attention_type": "MLA",
      "architectures": "GlmMoeDsaForCausalLM",
      "attention_note": "model_type glm_moe_dsa = MLA + DeepSeek Sparse Attention (DSA). architectures+index_topk -> SGLang serves with the 'dsa' backend; this bench benchmarks the DENSE absorbed-MLA decode via triton (the sparse indexer index_n_heads=32/index_head_dim=128/index_topk=2048 is NOT benchmarked) and records real_backend=dsa. rope_type 'default' -> no yarn mscale.",
      "hidden_size": 6144,
      "num_attention_heads": 64,
      "num_key_value_heads": 64,
      "qk_nope_head_dim": 192,
      "qk_rope_head_dim": 64,
      "head_dim_qk": 256,
      "v_head_dim": 256,
      "kv_lora_rank": 512,
      "q_lora_rank": 2048,
      "rope_scaling": {"rope_type": "default", "rope_theta": 1000000},
      "index_n_heads": 32,
      "index_head_dim": 128,
      "index_topk": 2048,
      "num_hidden_layers": 78,
      "max_position_embeddings": 202752
    }
  }
}