Skip to content
ATOM vLLM Nightly

ATOM vLLM Nightly #12

Sign in to view logs

ATOM vLLM Nightly

ATOM vLLM Nightly #12

Workflow file for this run

.github/workflows/atom-vllm-accuracy-validation.yaml at d69b063
name: ATOM vLLM Nightly
on:
schedule:
# Nightly at 00:00 Beijing time (16:00 UTC on the previous day)
- cron: '0 16 * * *'
workflow_dispatch:
# Manual runs use GitHub Actions' built-in branch dropdown ("Use workflow
# from") as the ATOM branch selector, so users do not need to type branch
# names.
inputs:
model_slot_1:
description: "Manual selection slot 1: choose an accuracy model case from .github/benchmark/oot_models_accuracy.json."
type: choice
default: none
options:
- none
- run_p0
- run_p1
- run_p2
- run_all
- Qwen3-235B-A22B-Instruct-2507-FP8 TP8+EP8
- Qwen3-Next-80B-A3B-Instruct-FP8 TP1
- Qwen3-Next-80B-A3B-Instruct-FP8 TP2
- Qwen3-Next-80B-A3B-Instruct-FP8 TP4
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP1
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP4
- Qwen3.5-397B-A17B-FP8 TP8
- Qwen3.5-397B-A17B-FP8 TP4
- Qwen3.5-397B-A17B TP8
- Qwen3.5-397B-A17B-MXFP4 TP4
- Meta-Llama-3.1-405B-Instruct-FP8 TP8
- Llama-3.1-8B-Instruct TP1
- Kimi-K2-Thinking-MXFP4 TP4
- Kimi-K2-Thinking-MXFP4 TP8
- Kimi-K2.5-MXFP4 TP4
- Kimi-K2.5-MXFP4 TP8
- Kimi-K3 TP8
- DeepSeek-R1-FP8 TP8
- DeepSeek-R1-FP8 DP8+EP8
- DeepSeek-R1-0528-MXFP4 TP8
- DeepSeek-V4-Pro TP8
- DeepSeek-V3.2-FP8 TP4
- DeepSeek-V3.2-FP8 TP8
- DeepSeek-V3.2-FP8 MTP TP4
- DeepSeek-V3.2-FP8 PTPC TP4
- gpt-oss-120b TP1
- gpt-oss-120b TP2
- gpt-oss-120b TP8
- MiniMax-M2.5 TP2
- MiniMax-M2.5 TP4
- GLM-4.7-FP8 TP4
- GLM-4.7-FP8 TP8
- GLM-4.7-FP8 MTP TP4
- GLM-4.7-FP8 MTP TP8
- GLM-5.1-FP8 TP8
- GLM-5.2-FP8 TP4
- GLM-5.2-FP8 MTP TP4
- GLM-5.2-MXFP4 TP4
- GLM-5.2-MXFP4 MTP TP4
model_slot_2:
description: "Manual selection slot 2: choose an accuracy model case from .github/benchmark/oot_models_accuracy.json."
type: choice
default: none
options:
- none
- run_p0
- run_p1
- run_p2
- run_all
- Qwen3-235B-A22B-Instruct-2507-FP8 TP8+EP8
- Qwen3-Next-80B-A3B-Instruct-FP8 TP1
- Qwen3-Next-80B-A3B-Instruct-FP8 TP2
- Qwen3-Next-80B-A3B-Instruct-FP8 TP4
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP1
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP4
- Qwen3.5-397B-A17B-FP8 TP8
- Qwen3.5-397B-A17B-FP8 TP4
- Qwen3.5-397B-A17B TP8
- Qwen3.5-397B-A17B-MXFP4 TP4
- Meta-Llama-3.1-405B-Instruct-FP8 TP8
- Llama-3.1-8B-Instruct TP1
- Kimi-K2-Thinking-MXFP4 TP4
- Kimi-K2-Thinking-MXFP4 TP8
- Kimi-K2.5-MXFP4 TP4
- Kimi-K2.5-MXFP4 TP8
- Kimi-K3 TP8
- DeepSeek-R1-FP8 TP8
- DeepSeek-R1-FP8 DP8+EP8
- DeepSeek-R1-0528-MXFP4 TP8
- DeepSeek-V4-Pro TP8
- DeepSeek-V3.2-FP8 TP4
- DeepSeek-V3.2-FP8 TP8
- DeepSeek-V3.2-FP8 MTP TP4
- DeepSeek-V3.2-FP8 PTPC TP4
- gpt-oss-120b TP1
- gpt-oss-120b TP2
- gpt-oss-120b TP8
- MiniMax-M2.5 TP2
- MiniMax-M2.5 TP4
- GLM-4.7-FP8 TP4
- GLM-4.7-FP8 TP8
- GLM-4.7-FP8 MTP TP4
- GLM-4.7-FP8 MTP TP8
- GLM-5.1-FP8 TP8
- GLM-5.2-FP8 TP4
- GLM-5.2-FP8 MTP TP4
- GLM-5.2-MXFP4 TP4
- GLM-5.2-MXFP4 MTP TP4
model_slot_3:
description: "Manual selection slot 3: choose an accuracy model case from .github/benchmark/oot_models_accuracy.json."
type: choice
default: none
options:
- none
- run_p0
- run_p1
- run_p2
- run_all
- Qwen3-235B-A22B-Instruct-2507-FP8 TP8+EP8
- Qwen3-Next-80B-A3B-Instruct-FP8 TP1
- Qwen3-Next-80B-A3B-Instruct-FP8 TP2
- Qwen3-Next-80B-A3B-Instruct-FP8 TP4
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP1
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP4
- Qwen3.5-397B-A17B-FP8 TP8
- Qwen3.5-397B-A17B-FP8 TP4
- Qwen3.5-397B-A17B TP8
- Qwen3.5-397B-A17B-MXFP4 TP4
- Meta-Llama-3.1-405B-Instruct-FP8 TP8
- Llama-3.1-8B-Instruct TP1
- Kimi-K2-Thinking-MXFP4 TP4
- Kimi-K2-Thinking-MXFP4 TP8
- Kimi-K2.5-MXFP4 TP4
- Kimi-K2.5-MXFP4 TP8
- Kimi-K3 TP8
- DeepSeek-R1-FP8 TP8
- DeepSeek-R1-FP8 DP8+EP8
- DeepSeek-R1-0528-MXFP4 TP8
- DeepSeek-V4-Pro TP8
- DeepSeek-V3.2-FP8 TP4
- DeepSeek-V3.2-FP8 TP8
- DeepSeek-V3.2-FP8 MTP TP4
- DeepSeek-V3.2-FP8 PTPC TP4
- gpt-oss-120b TP1
- gpt-oss-120b TP2
- gpt-oss-120b TP8
- MiniMax-M2.5 TP2
- MiniMax-M2.5 TP4
- GLM-4.7-FP8 TP4
- GLM-4.7-FP8 TP8
- GLM-4.7-FP8 MTP TP4
- GLM-4.7-FP8 MTP TP8
- GLM-5.1-FP8 TP8
- GLM-5.2-FP8 TP4
- GLM-5.2-FP8 MTP TP4
- GLM-5.2-MXFP4 TP4
- GLM-5.2-MXFP4 MTP TP4
model_slot_4:
description: "Manual selection slot 4: choose an accuracy model case from .github/benchmark/oot_models_accuracy.json."
type: choice
default: none
options:
- none
- run_p0
- run_p1
- run_p2
- run_all
- Qwen3-235B-A22B-Instruct-2507-FP8 TP8+EP8
- Qwen3-Next-80B-A3B-Instruct-FP8 TP1
- Qwen3-Next-80B-A3B-Instruct-FP8 TP2
- Qwen3-Next-80B-A3B-Instruct-FP8 TP4
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP1
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP4
- Qwen3.5-397B-A17B-FP8 TP8
- Qwen3.5-397B-A17B-FP8 TP4
- Qwen3.5-397B-A17B TP8
- Qwen3.5-397B-A17B-MXFP4 TP4
- Meta-Llama-3.1-405B-Instruct-FP8 TP8
- Llama-3.1-8B-Instruct TP1
- Kimi-K2-Thinking-MXFP4 TP4
- Kimi-K2-Thinking-MXFP4 TP8
- Kimi-K2.5-MXFP4 TP4
- Kimi-K2.5-MXFP4 TP8
- Kimi-K3 TP8
- DeepSeek-R1-FP8 TP8
- DeepSeek-R1-FP8 DP8+EP8
- DeepSeek-R1-0528-MXFP4 TP8
- DeepSeek-V4-Pro TP8
- DeepSeek-V3.2-FP8 TP4
- DeepSeek-V3.2-FP8 TP8
- DeepSeek-V3.2-FP8 MTP TP4
- DeepSeek-V3.2-FP8 PTPC TP4
- gpt-oss-120b TP1
- gpt-oss-120b TP2
- gpt-oss-120b TP8
- MiniMax-M2.5 TP2
- MiniMax-M2.5 TP4
- GLM-4.7-FP8 TP4
- GLM-4.7-FP8 TP8
- GLM-4.7-FP8 MTP TP4
- GLM-4.7-FP8 MTP TP8
- GLM-5.1-FP8 TP8
- GLM-5.2-FP8 TP4
- GLM-5.2-FP8 MTP TP4
- GLM-5.2-MXFP4 TP4
- GLM-5.2-MXFP4 MTP TP4
model_slot_5:
description: "Manual selection slot 5: choose an accuracy model case from .github/benchmark/oot_models_accuracy.json."
type: choice
default: none
options:
- none
- run_p0
- run_p1
- run_p2
- run_all
- Qwen3-235B-A22B-Instruct-2507-FP8 TP8+EP8
- Qwen3-Next-80B-A3B-Instruct-FP8 TP1
- Qwen3-Next-80B-A3B-Instruct-FP8 TP2
- Qwen3-Next-80B-A3B-Instruct-FP8 TP4
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP1
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP4
- Qwen3.5-397B-A17B-FP8 TP8
- Qwen3.5-397B-A17B-FP8 TP4
- Qwen3.5-397B-A17B TP8
- Qwen3.5-397B-A17B-MXFP4 TP4
- Meta-Llama-3.1-405B-Instruct-FP8 TP8
- Llama-3.1-8B-Instruct TP1
- Kimi-K2-Thinking-MXFP4 TP4
- Kimi-K2-Thinking-MXFP4 TP8
- Kimi-K2.5-MXFP4 TP4
- Kimi-K2.5-MXFP4 TP8
- Kimi-K3 TP8
- DeepSeek-R1-FP8 TP8
- DeepSeek-R1-FP8 DP8+EP8
- DeepSeek-R1-0528-MXFP4 TP8
- DeepSeek-V4-Pro TP8
- DeepSeek-V3.2-FP8 TP4
- DeepSeek-V3.2-FP8 TP8
- DeepSeek-V3.2-FP8 MTP TP4
- DeepSeek-V3.2-FP8 PTPC TP4
- gpt-oss-120b TP1
- gpt-oss-120b TP2
- gpt-oss-120b TP8
- MiniMax-M2.5 TP2
- MiniMax-M2.5 TP4
- GLM-4.7-FP8 TP4
- GLM-4.7-FP8 TP8
- GLM-4.7-FP8 MTP TP4
- GLM-4.7-FP8 MTP TP8
- GLM-5.1-FP8 TP8
- GLM-5.2-FP8 TP4
- GLM-5.2-FP8 MTP TP4
- GLM-5.2-MXFP4 TP4
- GLM-5.2-MXFP4 MTP TP4
model_slot_6:
description: "Manual selection slot 6: choose an accuracy model case from .github/benchmark/oot_models_accuracy.json."
type: choice
default: none
options:
- none
- run_p0
- run_p1
- run_p2
- run_all
- Qwen3-235B-A22B-Instruct-2507-FP8 TP8+EP8
- Qwen3-Next-80B-A3B-Instruct-FP8 TP1
- Qwen3-Next-80B-A3B-Instruct-FP8 TP2
- Qwen3-Next-80B-A3B-Instruct-FP8 TP4
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP1
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP4
- Qwen3.5-397B-A17B-FP8 TP8
- Qwen3.5-397B-A17B-FP8 TP4
- Qwen3.5-397B-A17B TP8
- Qwen3.5-397B-A17B-MXFP4 TP4
- Meta-Llama-3.1-405B-Instruct-FP8 TP8
- Llama-3.1-8B-Instruct TP1
- Kimi-K2-Thinking-MXFP4 TP4
- Kimi-K2-Thinking-MXFP4 TP8
- Kimi-K2.5-MXFP4 TP4
- Kimi-K2.5-MXFP4 TP8
- Kimi-K3 TP8
- DeepSeek-R1-FP8 TP8
- DeepSeek-R1-FP8 DP8+EP8
- DeepSeek-R1-0528-MXFP4 TP8
- DeepSeek-V4-Pro TP8
- DeepSeek-V3.2-FP8 TP4
- DeepSeek-V3.2-FP8 TP8
- DeepSeek-V3.2-FP8 MTP TP4
- DeepSeek-V3.2-FP8 PTPC TP4
- gpt-oss-120b TP1
- gpt-oss-120b TP2
- gpt-oss-120b TP8
- MiniMax-M2.5 TP2
- MiniMax-M2.5 TP4
- GLM-4.7-FP8 TP4
- GLM-4.7-FP8 TP8
- GLM-4.7-FP8 MTP TP4
- GLM-4.7-FP8 MTP TP8
- GLM-5.1-FP8 TP8
- GLM-5.2-FP8 TP4
- GLM-5.2-FP8 MTP TP4
- GLM-5.2-MXFP4 TP4
- GLM-5.2-MXFP4 MTP TP4
model_slot_7:
description: "Manual selection slot 7: choose an accuracy model case from .github/benchmark/oot_models_accuracy.json."
type: choice
default: none
options:
- none
- run_p0
- run_p1
- run_p2
- run_all
- Qwen3-235B-A22B-Instruct-2507-FP8 TP8+EP8
- Qwen3-Next-80B-A3B-Instruct-FP8 TP1
- Qwen3-Next-80B-A3B-Instruct-FP8 TP2
- Qwen3-Next-80B-A3B-Instruct-FP8 TP4
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP1
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP4
- Qwen3.5-397B-A17B-FP8 TP8
- Qwen3.5-397B-A17B-FP8 TP4
- Qwen3.5-397B-A17B TP8
- Qwen3.5-397B-A17B-MXFP4 TP4
- Meta-Llama-3.1-405B-Instruct-FP8 TP8
- Llama-3.1-8B-Instruct TP1
- Kimi-K2-Thinking-MXFP4 TP4
- Kimi-K2-Thinking-MXFP4 TP8
- Kimi-K2.5-MXFP4 TP4
- Kimi-K2.5-MXFP4 TP8
- Kimi-K3 TP8
- DeepSeek-R1-FP8 TP8
- DeepSeek-R1-FP8 DP8+EP8
- DeepSeek-R1-0528-MXFP4 TP8
- DeepSeek-V4-Pro TP8
- DeepSeek-V3.2-FP8 TP4
- DeepSeek-V3.2-FP8 TP8
- DeepSeek-V3.2-FP8 MTP TP4
- DeepSeek-V3.2-FP8 PTPC TP4
- gpt-oss-120b TP1
- gpt-oss-120b TP2
- gpt-oss-120b TP8
- MiniMax-M2.5 TP2
- MiniMax-M2.5 TP4
- GLM-4.7-FP8 TP4
- GLM-4.7-FP8 TP8
- GLM-4.7-FP8 MTP TP4
- GLM-4.7-FP8 MTP TP8
- GLM-5.1-FP8 TP8
- GLM-5.2-FP8 TP4
- GLM-5.2-FP8 MTP TP4
- GLM-5.2-MXFP4 TP4
- GLM-5.2-MXFP4 MTP TP4
model_slot_8:
description: "Manual selection slot 8: choose an accuracy model case from .github/benchmark/oot_models_accuracy.json."
type: choice
default: none
options:
- none
- run_p0
- run_p1
- run_p2
- run_all
- Qwen3-235B-A22B-Instruct-2507-FP8 TP8+EP8
- Qwen3-Next-80B-A3B-Instruct-FP8 TP1
- Qwen3-Next-80B-A3B-Instruct-FP8 TP2
- Qwen3-Next-80B-A3B-Instruct-FP8 TP4
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP1
- Qwen3-Next-80B-A3B-Instruct-FP8-MTP TP4
- Qwen3.5-397B-A17B-FP8 TP8
- Qwen3.5-397B-A17B-FP8 TP4
- Qwen3.5-397B-A17B TP8
- Qwen3.5-397B-A17B-MXFP4 TP4
- Meta-Llama-3.1-405B-Instruct-FP8 TP8
- Llama-3.1-8B-Instruct TP1
- Kimi-K2-Thinking-MXFP4 TP4
- Kimi-K2-Thinking-MXFP4 TP8
- Kimi-K2.5-MXFP4 TP4
- Kimi-K2.5-MXFP4 TP8
- Kimi-K3 TP8
- DeepSeek-R1-FP8 TP8
- DeepSeek-R1-FP8 DP8+EP8
- DeepSeek-R1-0528-MXFP4 TP8
- DeepSeek-V4-Pro TP8
- DeepSeek-V3.2-FP8 TP4
- DeepSeek-V3.2-FP8 TP8
- DeepSeek-V3.2-FP8 MTP TP4
- DeepSeek-V3.2-FP8 PTPC TP4
- gpt-oss-120b TP1
- gpt-oss-120b TP2
- gpt-oss-120b TP8
- MiniMax-M2.5 TP2
- MiniMax-M2.5 TP4
- GLM-4.7-FP8 TP4
- GLM-4.7-FP8 TP8
- GLM-4.7-FP8 MTP TP4
- GLM-4.7-FP8 MTP TP8
- GLM-5.1-FP8 TP8
- GLM-5.2-FP8 TP4
- GLM-5.2-FP8 MTP TP4
- GLM-5.2-MXFP4 TP4
- GLM-5.2-MXFP4 MTP TP4
oot_image:
description: "Optional atom-vLLM image override. Leave empty to keep the existing nightly/manual build behavior."
required: false
type: string
default: ""
rebuild_atom_base_from_dockerfile:
description: "Full build image before validation"
required: false
type: boolean
default: false
upload_accuracy_to_dashboard:
description: "Optional: upload OOT accuracy results to dashboard after this manual run"
required: false
type: boolean
default: false
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: false
env:
ATOM_BASE_NIGHTLY_IMAGE: rocm/atom-dev:latest
NIGHTLY_OOT_IMAGE_TAG: rocm/atom-dev:vllm-latest
VALIDATION_IMAGE_REPO: rocm/atom-dev
GPU_ARCH: "gfx942;gfx950"
AITER_REPO: "https://github.com/ROCm/aiter.git"
AITER_COMMIT: "HEAD"
jobs:
# ---------- OOT full validation path (schedule / workflow_dispatch) ----------
prepare-oot-image:
name: Prepare validation image
runs-on: build-only-atom
permissions:
actions: read
contents: read
outputs:
oot_image_tag: ${{ steps.meta.outputs.oot_image_tag }}
cleanup_temp_image: ${{ steps.meta.outputs.cleanup_temp_image }}
model_matrix_slot_1: ${{ steps.meta.outputs.model_matrix_slot_1 }}
model_matrix_slot_2: ${{ steps.meta.outputs.model_matrix_slot_2 }}
model_matrix_slot_3: ${{ steps.meta.outputs.model_matrix_slot_3 }}
slot_1_has_models: ${{ steps.meta.outputs.slot_1_has_models }}
slot_2_has_models: ${{ steps.meta.outputs.slot_2_has_models }}
slot_3_has_models: ${{ steps.meta.outputs.slot_3_has_models }}
scheduled_beijing_date: ${{ steps.meta.outputs.scheduled_beijing_date }}
steps:
- name: Checkout selected ATOM branch
uses: actions/checkout@v6
- name: Load vLLM version
run: cat atom/plugin/vllm/vllm-version.env >> "$GITHUB_ENV"
- name: Docker Login
if: ${{ github.event_name == 'workflow_dispatch' }}
run: |
echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
- name: Resolve image source and model matrix
id: meta
env:
MODEL_SLOT_1: ${{ inputs.model_slot_1 || 'none' }}
MODEL_SLOT_2: ${{ inputs.model_slot_2 || 'none' }}
MODEL_SLOT_3: ${{ inputs.model_slot_3 || 'none' }}
MODEL_SLOT_4: ${{ inputs.model_slot_4 || 'none' }}
MODEL_SLOT_5: ${{ inputs.model_slot_5 || 'none' }}
MODEL_SLOT_6: ${{ inputs.model_slot_6 || 'none' }}
MODEL_SLOT_7: ${{ inputs.model_slot_7 || 'none' }}
MODEL_SLOT_8: ${{ inputs.model_slot_8 || 'none' }}
GITHUB_TOKEN: ${{ github.token }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
INPUT_OOT_IMAGE: ${{ inputs.oot_image || '' }}
REBUILD_ATOM_BASE_FROM_DOCKERFILE: ${{ inputs.rebuild_atom_base_from_dockerfile }}
run: |
set -euo pipefail
if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then
SCHEDULE_CREATED_AT="$(
python3 - <<'PY'
import json
import os
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com")
repository = os.environ["GITHUB_REPOSITORY"]
run_id = os.environ["GITHUB_RUN_ID"]
token = os.environ.get("GITHUB_TOKEN", "")
request = Request(
f"{api_url}/repos/{repository}/actions/runs/{run_id}",
headers={
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {token}",
"User-Agent": "atom-vllm-accuracy-nightly-priority",
},
)
try:
with urlopen(request, timeout=30) as response:
payload = json.load(response)
except HTTPError as exc:
detail = exc.read().decode("utf-8", errors="replace").strip()
raise RuntimeError(
f"Failed to fetch workflow run metadata: HTTP {exc.code} {detail}"
) from exc
except URLError as exc:
raise RuntimeError(
f"Failed to reach GitHub API for workflow run metadata: {exc}"
) from exc
created_at = payload.get("created_at")
if not created_at:
raise RuntimeError("Workflow run metadata did not include created_at")
print(created_at)
PY
)"
export SCHEDULE_CREATED_AT
else
export SCHEDULE_CREATED_AT=""
fi
# Keep .github/benchmark/oot_models_accuracy.json as the single source
# of truth for OOT full-validation coverage and runtime configuration.
python3 - <<'PY' >> "$GITHUB_OUTPUT"
import json
import os
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
event = os.environ["GITHUB_EVENT_NAME"]
models_path = Path(".github/benchmark/oot_models_accuracy.json")
raw_models = json.loads(models_path.read_text(encoding="utf-8"))
if not isinstance(raw_models, list):
raise SystemExit("OOT accuracy model catalog must be a JSON list.")
required_fields = {
"model_name",
"model_path",
"extra_args",
"env_vars",
"runner",
"priority",
"accuracy_test_threshold",
}
def normalize_model(raw_model, index):
if not isinstance(raw_model, dict):
raise SystemExit(f"Model catalog entry #{index} must be an object.")
model = dict(raw_model)
# Keep a short compatibility bridge while older local branches still use
# the former dashboard-only field names. New entries should use snake_case.
if "extra_args" not in model and "extraArgs" in model:
model["extra_args"] = model.pop("extraArgs")
else:
model.pop("extraArgs", None)
if "accuracy_test_threshold" not in model and "accuracy_threshold" in model:
model["accuracy_test_threshold"] = model["accuracy_threshold"]
missing = sorted(required_fields - model.keys())
if missing:
name = model.get("model_name", f"entry #{index}")
raise SystemExit(
f"OOT accuracy model {name!r} is missing required fields: "
+ ", ".join(missing)
)
model["model_name"] = str(model["model_name"])
model["model_path"] = str(model["model_path"])
model["extra_args"] = str(model.get("extra_args", ""))
model["env_vars"] = str(model.get("env_vars", ""))
model["runner"] = str(model["runner"])
model["test_level"] = str(model.get("test_level", "nightly"))
model["priority"] = str(model["priority"])
if model["priority"] not in {"P0", "P1", "P2"}:
raise SystemExit(
f"OOT accuracy model {model['model_name']!r} has invalid priority "
f"{model['priority']!r}; expected P0, P1, or P2."
)
if "lm_eval_num_fewshot" in model:
model["lm_eval_num_fewshot"] = int(model["lm_eval_num_fewshot"])
try:
model["accuracy_test_threshold"] = float(model["accuracy_test_threshold"])
except (TypeError, ValueError) as exc:
raise SystemExit(
f"OOT accuracy model {model['model_name']!r} has non-numeric "
"accuracy_test_threshold."
) from exc
return model
models = []
seen_model_names = set()
for index, raw_model in enumerate(raw_models, start=1):
model = normalize_model(raw_model, index)
if model["model_name"] in seen_model_names:
raise SystemExit(
f"Duplicate OOT accuracy model_name: {model['model_name']!r}"
)
seen_model_names.add(model["model_name"])
models.append(model)
def matrix_entry(model):
return dict(model)
selected = []
if event == "workflow_dispatch":
by_name = {model["model_name"]: model for model in models}
group_selections = {
"run_p0": [model for model in models if model["priority"] == "P0"],
"run_p1": [model for model in models if model["priority"] == "P1"],
"run_p2": [model for model in models if model["priority"] == "P2"],
"run_all": models,
}
seen = set()
def add_model(model):
name = model["model_name"]
if name in seen:
return
selected.append(matrix_entry(model))
seen.add(name)
for slot_idx in range(1, 9):
label = os.environ.get(f"MODEL_SLOT_{slot_idx}", "").strip()
if not label or label == "none":
continue
if label in group_selections:
for model in group_selections[label]:
add_model(model)
continue
if label not in by_name:
print(
f"Unknown model_slot_{slot_idx} selection: {label}",
file=sys.stderr,
)
print("Available model_slot values:", file=sys.stderr)
for value in ["none", *group_selections, *by_name]:
print(f" - {value}", file=sys.stderr)
sys.exit(1)
add_model(by_name[label])
else:
selected = [
matrix_entry(model)
for model in models
if model.get("test_level") == "nightly"
]
if event == "workflow_dispatch" and not selected:
print(
"No models selected for manual run. Choose at least one model_slot value.",
file=sys.stderr,
)
sys.exit(1)
beijing_tz = timezone(timedelta(hours=8))
validation_beijing_now = os.environ.get("VALIDATION_BEIJING_NOW", "")
if validation_beijing_now:
beijing_now = datetime.fromisoformat(validation_beijing_now)
if beijing_now.tzinfo is None:
beijing_now = beijing_now.replace(tzinfo=beijing_tz)
else:
beijing_now = beijing_now.astimezone(beijing_tz)
elif event == "schedule":
created_at = os.environ.get("SCHEDULE_CREATED_AT", "")
if not created_at:
raise SystemExit(
"Scheduled validation requires workflow run created_at metadata."
)
beijing_now = datetime.fromisoformat(
created_at.replace("Z", "+00:00")
).astimezone(beijing_tz)
else:
beijing_now = datetime.now(timezone.utc).astimezone(beijing_tz)
if event == "schedule":
run_date = beijing_now.date().isoformat()
else:
run_date = ""
order = ("P1", "P0", "P2")
if event != "schedule" or beijing_now.weekday() < 5:
order = ("P0", "P1", "P2")
priority_slots = {priority: [] for priority in ("P0", "P1", "P2")}
for model in selected:
priority_slots[model["priority"]].append(model)
for slot_idx, priority in enumerate(order, start=1):
matrix = {"include": priority_slots[priority]}
print(f"model_matrix_slot_{slot_idx}={json.dumps(matrix, separators=(',', ':'))}")
print(f"slot_{slot_idx}_has_models={'true' if priority_slots[priority] else 'false'}")
print(f"slot_{slot_idx}_priority={priority}")
print(f"scheduled_beijing_date={run_date}")
PY
REBUILD_FROM_DOCKERFILE="${REBUILD_ATOM_BASE_FROM_DOCKERFILE:-false}"
if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then
if [ "${REBUILD_FROM_DOCKERFILE}" = "true" ]; then
echo "Manual run selected full ATOM base rebuild from docker/atom_release.dockerfile"
else
echo "Manual run selected fast overlay rebuild from ${ATOM_BASE_NIGHTLY_IMAGE}"
fi
fi
if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then
if RESOLUTION_JSON="$(
python3 .github/scripts/resolve_atom_image.py \
--repository "${VALIDATION_IMAGE_REPO}" \
--reference-tag vllm-latest
)"; then
RESOLVED_OOT_IMAGE="$(
RESOLUTION_JSON="${RESOLUTION_JSON}" python3 - <<'PY'
import json
import os
resolution = json.loads(os.environ["RESOLUTION_JSON"])
print(resolution["resolved_image"])
PY
)"
echo "Scheduled run resolved published OOT image: ${RESOLVED_OOT_IMAGE}"
else
echo "::warning::Failed to resolve ${NIGHTLY_OOT_IMAGE_TAG} to an immutable reference; falling back to the floating tag."
RESOLVED_OOT_IMAGE="${NIGHTLY_OOT_IMAGE_TAG}"
fi
echo "oot_image_tag=${RESOLVED_OOT_IMAGE}" >> "$GITHUB_OUTPUT"
echo "cleanup_temp_image=false" >> "$GITHUB_OUTPUT"
echo "manual_base_image_tag=" >> "$GITHUB_OUTPUT"
echo "rebuilt_atom_base_image_tag=" >> "$GITHUB_OUTPUT"
exit 0
fi
if [ -n "${INPUT_OOT_IMAGE}" ]; then
echo "Manual run will use user-provided atom-vLLM image: ${INPUT_OOT_IMAGE}"
echo "oot_image_tag=${INPUT_OOT_IMAGE}" >> "$GITHUB_OUTPUT"
echo "cleanup_temp_image=false" >> "$GITHUB_OUTPUT"
echo "manual_base_image_tag=" >> "$GITHUB_OUTPUT"
exit 0
fi
SHORT_SHA="$(git rev-parse --short=12 HEAD)"
MANUAL_BASE_IMAGE_TAG="atom_oot_manual_base:${GITHUB_RUN_ID}-${SHORT_SHA}"
REBUILT_ATOM_BASE_IMAGE_TAG="atom_oot_rebuilt_base:${GITHUB_RUN_ID}-${SHORT_SHA}"
OOT_IMAGE_TAG="${VALIDATION_IMAGE_REPO}:vllm-full-manual-${GITHUB_RUN_ID}-${SHORT_SHA}"
BUILD_CONTEXT="$(mktemp -d)"
trap 'rm -rf "${BUILD_CONTEXT}"' EXIT
echo "oot_image_tag=${OOT_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
echo "cleanup_temp_image=true" >> "$GITHUB_OUTPUT"
if [ "${REBUILD_FROM_DOCKERFILE}" = "true" ]; then
echo "manual_base_image_tag=" >> "$GITHUB_OUTPUT"
echo "rebuilt_atom_base_image_tag=${REBUILT_ATOM_BASE_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
else
echo "manual_base_image_tag=${MANUAL_BASE_IMAGE_TAG}" >> "$GITHUB_OUTPUT"
echo "rebuilt_atom_base_image_tag=" >> "$GITHUB_OUTPUT"
fi
git archive --format=tar HEAD | tar -xf - -C "${BUILD_CONTEXT}"
if [ "${REBUILD_FROM_DOCKERFILE}" = "true" ]; then
echo "Manual run rebuilds ATOM base and OOT image from branch ${GITHUB_REF_NAME}"
DOCKER_BUILDKIT=1 docker build --pull --network=host --no-cache \
-t "${REBUILT_ATOM_BASE_IMAGE_TAG}" \
--build-arg GPU_ARCH="${GPU_ARCH}" \
--build-arg AITER_REPO="${AITER_REPO}" \
--build-arg AITER_COMMIT="${AITER_COMMIT}" \
--build-arg MAX_JOBS=64 \
--build-arg ATOM_REPO="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" \
--build-arg ATOM_COMMIT="${GITHUB_SHA}" \
-f "${BUILD_CONTEXT}/docker/atom_release.dockerfile" \
"${BUILD_CONTEXT}"
DOCKER_BUILDKIT=1 docker build --network=host --no-cache \
-t "${OOT_IMAGE_TAG}" \
--build-arg OOT_BASE_IMAGE="${REBUILT_ATOM_BASE_IMAGE_TAG}" \
--build-arg MAX_JOBS=64 \
--build-arg VLLM_COMMIT="${VLLM_COMMIT}" \
--build-arg INSTALL_LM_EVAL=1 \
--build-arg INSTALL_FASTSAFETENSORS=1 \
-f "${BUILD_CONTEXT}/docker/vllm_release.dockerfile" \
"${BUILD_CONTEXT}"
.github/scripts/docker_push_retry.sh "${OOT_IMAGE_TAG}"
exit 0
fi
cat <<'EOF' > "${BUILD_CONTEXT}/Dockerfile.mod"
ARG ATOM_BASE_NIGHTLY_IMAGE
FROM ${ATOM_BASE_NIGHTLY_IMAGE}
ARG GPU_ARCH
ARG AITER_REPO
ARG AITER_COMMIT
RUN pip install -U lm-eval[api]
RUN pip show lm-eval || true
RUN pip install hf_transfer
RUN pip show hf_transfer || true
RUN echo "=== Aiter version BEFORE uninstall ===" && pip show amd-aiter || true
RUN pip uninstall -y amd-aiter
RUN pip install --upgrade "pybind11>=3.0.1"
RUN pip show pybind11
RUN rm -rf /app/aiter-test
RUN git clone --depth 1 "${AITER_REPO}" /app/aiter-test && \
cd /app/aiter-test && \
git checkout "${AITER_COMMIT}" && \
git submodule sync && git submodule update --init --recursive && \
MAX_JOBS=64 PREBUILD_KERNELS=0 GPU_ARCHS="${GPU_ARCH}" python3 setup.py develop
RUN echo "=== Aiter version AFTER installation ===" && pip show amd-aiter || true
RUN echo "=== ATOM version BEFORE uninstall ===" && pip show atom || true
RUN pip uninstall -y atom
RUN rm -rf /app/ATOM
COPY . /app/ATOM
RUN cd /app/ATOM && pip install -e .
RUN echo "=== ATOM version AFTER installation ===" && pip show atom || true
EOF
echo "Manual run rebuilds OOT image from branch ${GITHUB_REF_NAME}"
docker build --pull --network=host --no-cache \
-t "${MANUAL_BASE_IMAGE_TAG}" \
--build-arg ATOM_BASE_NIGHTLY_IMAGE="${ATOM_BASE_NIGHTLY_IMAGE}" \
--build-arg GPU_ARCH="${GPU_ARCH}" \
--build-arg AITER_REPO="${AITER_REPO}" \
--build-arg AITER_COMMIT="${AITER_COMMIT}" \
-f "${BUILD_CONTEXT}/Dockerfile.mod" \
"${BUILD_CONTEXT}"
docker build --network=host --no-cache \
-t "${OOT_IMAGE_TAG}" \
--build-arg OOT_BASE_IMAGE="${MANUAL_BASE_IMAGE_TAG}" \
--build-arg MAX_JOBS=64 \
--build-arg VLLM_COMMIT="${VLLM_COMMIT}" \
--build-arg INSTALL_LM_EVAL=1 \
--build-arg INSTALL_FASTSAFETENSORS=1 \
-f "${BUILD_CONTEXT}/docker/vllm_release.dockerfile" \
"${BUILD_CONTEXT}"
.github/scripts/docker_push_retry.sh "${OOT_IMAGE_TAG}"
- name: Clean up builder images
if: ${{ always() && steps.meta.outputs.cleanup_temp_image == 'true' }}
run: |
for image in \
"${{ steps.meta.outputs.oot_image_tag }}" \
"${{ steps.meta.outputs.manual_base_image_tag }}" \
"${{ steps.meta.outputs.rebuilt_atom_base_image_tag }}"; do
if [ -n "${image}" ]; then
docker rmi "${image}" || true
fi
done
oot-model-accuracy-priority-1:
name: Model Accuracy ${{ matrix.priority }} (${{ matrix.model_name }})
needs: [prepare-oot-image]
if: ${{ needs.prepare-oot-image.outputs.slot_1_has_models == 'true' }}
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.prepare-oot-image.outputs.model_matrix_slot_1) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 240
permissions:
actions: read
contents: write
env:
CONTAINER_NAME: atom_vllm_oot_validation_${{ matrix.priority }}_${{ strategy.job-index }}
OOT_IMAGE_TAG: ${{ needs.prepare-oot-image.outputs.oot_image_tag }}
steps: &oot-model-accuracy-steps
- name: Check validation date window
id: validation-window
env:
SCHEDULED_BEIJING_DATE: ${{ needs.prepare-oot-image.outputs.scheduled_beijing_date }}
MIN_REMAINING_MINUTES: "150"
run: |
if [ "${GITHUB_EVENT_NAME}" != "schedule" ]; then
echo "should_run=true" >> "$GITHUB_OUTPUT"
echo "reason=manual run" >> "$GITHUB_OUTPUT"
exit 0
fi
python3 - <<'PY' >> "$GITHUB_OUTPUT"
import os
from datetime import datetime, timedelta, timezone
beijing_tz = timezone(timedelta(hours=8))
now = datetime.now(timezone.utc).astimezone(beijing_tz)
scheduled_date = os.environ.get("SCHEDULED_BEIJING_DATE", "")
min_remaining_minutes = int(os.environ.get("MIN_REMAINING_MINUTES", "150"))
current_date = now.date().isoformat()
end_of_day = now.replace(hour=23, minute=59, second=59, microsecond=0)
remaining_minutes = int((end_of_day - now).total_seconds() // 60)
should_run = (
current_date == scheduled_date
and remaining_minutes >= min_remaining_minutes
)
print(f"should_run={'true' if should_run else 'false'}")
print(f"current_beijing_date={current_date}")
print(f"remaining_minutes={remaining_minutes}")
if current_date != scheduled_date:
print(
f"reason=scheduled date {scheduled_date} no longer matches current Beijing date {current_date}"
)
elif remaining_minutes < min_remaining_minutes:
print(
f"reason=only {remaining_minutes} Beijing-day minutes remain, below required {min_remaining_minutes}"
)
else:
print("reason=same Beijing day with enough time remaining")
PY
echo "Validation window check:"
echo "- should_run=$(sed -n 's/^should_run=//p' "$GITHUB_OUTPUT")"
echo "- reason=$(sed -n 's/^reason=//p' "$GITHUB_OUTPUT")"
- name: Detect container engine
if: ${{ steps.validation-window.outputs.should_run == 'true' }}
run: |
if command -v podman > /dev/null 2>&1; then
echo "CONTAINER_ENGINE=podman" >> "$GITHUB_ENV"
echo "Container engine: podman"
elif docker info > /dev/null 2>&1; then
echo "CONTAINER_ENGINE=docker" >> "$GITHUB_ENV"
echo "Container engine: docker"
else
echo "ERROR: Neither docker nor podman is available on this runner."
exit 1
fi
- name: Checkout ATOM repo
if: ${{ steps.validation-window.outputs.should_run == 'true' }}
uses: actions/checkout@v6
- name: Docker Login
if: ${{ steps.validation-window.outputs.should_run == 'true' }}
run: |
REG="${OOT_IMAGE_TAG%%/*}"
if [[ "$REG" != *.* && "$REG" != localhost* && "$REG" != *:* ]]; then
REG="docker.io"
fi
echo "Logging in to registry: ${REG}"
echo "${{ secrets.DOCKER_PASSWORD }}" | $CONTAINER_ENGINE login "$REG" -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
- name: Set HF_TOKEN
if: ${{ steps.validation-window.outputs.should_run == 'true' }}
run: echo "HF_TOKEN=${HF_TOKEN:-${{ secrets.AMD_HF_TOKEN }}}" >> "$GITHUB_ENV"
- name: Pull OOT image
if: ${{ steps.validation-window.outputs.should_run == 'true' }}
run: |
set -euo pipefail
IMG="${OOT_IMAGE_TAG}"
if [[ "${CONTAINER_ENGINE}" == "podman" ]]; then
if [[ "${IMG}" != */* ]]; then
IMG="docker.io/library/${IMG}"
else
reg="${IMG%%/*}"
if [[ "${reg}" != *.* && "${reg}" != localhost* && "${reg}" != *:* ]]; then
IMG="docker.io/${IMG}"
fi
fi
fi
echo "OOT_IMAGE_REF=${IMG}" >> "$GITHUB_ENV"
echo "Pulling OOT image: ${IMG} (workflow tag: ${OOT_IMAGE_TAG})"
$CONTAINER_ENGINE pull "${IMG}"
- name: Prepare model cache mount
if: ${{ steps.validation-window.outputs.should_run == 'true' }}
run: |
MODEL_CACHE_ROOT=""
MODEL_CACHE_MOUNT=""
MODEL_CACHE_DESC="container-local /models (no host cache mount)"
MODEL_MOUNT_PATH="/models"
if [ -d "/mnt/dcgpuval" ]; then
MODEL_CACHE_ROOT="/mnt/dcgpuval"
elif [ -d "/shared/data/WRH/models" ]; then
MODEL_CACHE_ROOT="/shared/data/WRH/models"
elif [ -d "/shared_nfs/models" ]; then
MODEL_CACHE_ROOT="/shared_nfs/models"
elif [ -d "/it-share/models" ]; then
MODEL_CACHE_ROOT="/it-share/models"
elif [ -d "/models" ]; then
MODEL_CACHE_ROOT="/models"
elif [ -d "/data/models" ]; then
MODEL_CACHE_ROOT="/data/models"
fi
if [ -n "${MODEL_CACHE_ROOT}" ]; then
MODEL_CACHE_MOUNT="-v ${MODEL_CACHE_ROOT}:${MODEL_MOUNT_PATH}"
MODEL_CACHE_DESC="${MODEL_CACHE_ROOT} (shared host path)"
else
echo "No shared host model cache found; using container-local ${MODEL_MOUNT_PATH}."
fi
echo "Using model cache backend: ${MODEL_CACHE_DESC}"
echo "MODEL_CACHE_ROOT=${MODEL_CACHE_ROOT}" >> "$GITHUB_ENV"
echo "MODEL_CACHE_MOUNT=${MODEL_CACHE_MOUNT}" >> "$GITHUB_ENV"
echo "MODEL_CACHE_DESC=${MODEL_CACHE_DESC}" >> "$GITHUB_ENV"
echo "MODEL_MOUNT_PATH=${MODEL_MOUNT_PATH}" >> "$GITHUB_ENV"
- name: Clean up old containers
if: ${{ steps.validation-window.outputs.should_run == 'true' }}
run: |
containers=$($CONTAINER_ENGINE ps -q)
if [ -n "$containers" ]; then
$CONTAINER_ENGINE kill $containers || true
fi
$CONTAINER_ENGINE rm -f "$CONTAINER_NAME" 2>/dev/null || true
- name: Start validation container
if: ${{ steps.validation-window.outputs.should_run == 'true' }}
run: |
if [ -f "/etc/podinfo/gha-render-devices" ]; then
DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices)
else
DEVICE_FLAG="--device /dev/dri"
fi
if [ "${CONTAINER_ENGINE}" = "podman" ]; then
GROUP_ADD_FLAG="--group-add keep-groups"
else
GROUP_ADD_FLAG="--group-add video"
fi
MODEL_MOUNT="${MODEL_CACHE_MOUNT}"
echo "Using model cache backend: ${MODEL_CACHE_DESC}"
{
printf '%s\n' "${{ matrix.env_vars }}"
printf '%s\n' \
"AITER_LOG_LEVEL=ERROR" \
"AITER_TRITON_LOG_LEVEL=CRITICAL" \
"AITER_LOG_MORE=0"
} | sed '/^[[:space:]]*$/d; s/^[[:space:]]*//' > /tmp/oot_env_file.txt
$CONTAINER_ENGINE run -dt --device=/dev/kfd $DEVICE_FLAG \
-v "${GITHUB_WORKSPACE:-$PWD}":/workspace \
$MODEL_MOUNT \
-w /workspace \
--ipc=host $GROUP_ADD_FLAG \
--privileged \
--cap-add=SYS_PTRACE \
--security-opt seccomp=unconfined \
--ulimit memlock=-1 \
--ulimit stack=67108864 \
--env-file /tmp/oot_env_file.txt \
-e HF_TOKEN="${HF_TOKEN:-}" \
--name "$CONTAINER_NAME" \
"${OOT_IMAGE_REF:-${OOT_IMAGE_TAG}}"
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Collect GPU info (inside container)
if: ${{ steps.validation-window.outputs.should_run == 'true' }}
id: gpu-info
run: bash .github/scripts/collect_gpu_info.sh "$CONTAINER_NAME" "$CONTAINER_ENGINE" "${{ matrix.runner }}"
- name: Resolve local model path
if: ${{ steps.validation-window.outputs.should_run == 'true' }}
run: |
set -euo pipefail
if [ -z "${MODEL_CACHE_MOUNT}" ]; then
echo "OOT_MODEL_AVAILABLE_LOCALLY=false" >> "$GITHUB_ENV"
echo "No shared model cache mount; local model resolution skipped."
exit 0
fi
if $CONTAINER_ENGINE exec \
-e MODEL_PATH="${{ matrix.model_path }}" \
"$CONTAINER_NAME" \
bash -lc '
set -euo pipefail
python3 - <<'"'"'PY'"'"'
import os
from pathlib import Path
model_path = os.environ["MODEL_PATH"].strip().strip("/")
normalized = model_path.rstrip("/")
basename = Path(normalized).name
alias_map = {
"Qwen/Qwen3-235B-A22B-Instruct-2507-FP8": [
"/models/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8",
"/models/Qwen3-235B-A22B-Instruct-2507-FP8",
],
"Qwen/Qwen3-Next-80B-A3B-Instruct-FP8": [
"/models/Qwen/Qwen3-Next-80B-A3B-Instruct-FP8",
"/models/Qwen3-Next-80B-A3B-Instruct-FP8",
],
"Qwen/Qwen3.5-397B-A17B-FP8": [
"/models/Qwen/Qwen3.5-397B-A17B-FP8",
"/models/Qwen3.5-397B-A17B-FP8",
],
"Qwen/Qwen3.5-397B-A17B": [
"/models/Qwen/Qwen3.5-397B-A17B",
"/models/Qwen3.5-397B-A17B",
],
"amd/Qwen3.5-397B-A17B-MXFP4": [
"/models/amd/Qwen3.5-397B-A17B-MXFP4",
"/models/Qwen/Qwen3.5-397B-A17B-MXFP4",
"/models/Qwen3.5-397B-A17B-MXFP4",
],
"Meta-Llama-3.1-405B-Instruct-FP8": [
"/models/Meta-Llama-3.1-405B-Instruct-FP8",
"/models/facebook/Meta-Llama-3.1-405B-Instruct-FP8",
],
"meta-llama/Llama-3.1-8B-Instruct": [
"/models/meta-llama/Llama-3.1-8B-Instruct",
"/models/Llama-3.1-8B-Instruct",
],
"amd/Kimi-K2-Thinking-MXFP4-AttnFP8": [
"/models/amd/Kimi-K2-Thinking-MXFP4-AttnFP8",
"/models/kimi/Kimi-K2-Thinking-MXFP4-AttnFP8",
"/models/Kimi-K2-Thinking-MXFP4-AttnFP8",
],
"amd/Kimi-K2.5-MXFP4-AttnFP8": [
"/models/amd/Kimi-K2.5-MXFP4-AttnFP8",
"/models/kimi/Kimi-K2.5-MXFP4-AttnFP8",
"/models/Kimi-K2.5-MXFP4-AttnFP8",
],
"deepseek-ai/DeepSeek-R1-0528": [
"/models/deepseek-ai/DeepSeek-R1-0528",
"/models/DeepSeek-R1-0528",
],
"amd/DeepSeek-R1-0528-MXFP4-MTP-MoEFP4": [
"/models/amd/DeepSeek-R1-0528-MXFP4-MTP-MoEFP4",
"/models/deepseek-ai/DeepSeek-R1-0528-MXFP4-MTP-MoEFP4",
"/models/DeepSeek-R1-0528-MXFP4-MTP-MoEFP4",
],
"deepseek-ai/DeepSeek-V4-Pro": [
"/models/deepseek-ai/DeepSeek-V4-Pro",
"/models/DeepSeek-V4-Pro",
],
"deepseek-ai/DeepSeek-V3.2": [
"/models/deepseek-ai/DeepSeek-V3.2",
"/models/DeepSeek-V3.2",
],
"openai/gpt-oss-120b": [
"/models/openai/gpt-oss-120b",
"/models/gpt-oss-120b",
],
"MiniMaxAI/MiniMax-M2.5": [
"/models/MiniMaxAI/MiniMax-M2.5",
"/models/MiniMax-M2.5",
],
"zai-org/GLM-4.7-FP8": [
"/models/zai-org/GLM-4.7-FP8",
"/models/GLM/GLM-4.7-FP8",
"/models/GLM-4.7-FP8",
],
"zai-org/GLM-5.1-FP8": [
"/models/zai-org/GLM-5.1-FP8",
"/models/GLM/GLM-5.1-FP8",
"/models/GLM-5.1-FP8",
],
"zai-org/GLM-5.2-FP8": [
"/models/zai-org/GLM-5.2-FP8",
"/models/GLM/GLM-5.2-FP8",
"/models/GLM-5.2-FP8",
],
"amd/GLM-5.2-MXFP4": [
"/models/amd/GLM-5.2-MXFP4",
"/models/GLM/GLM-5.2-MXFP4",
"/models/GLM-5.2-MXFP4",
],
}
candidates = []
if normalized:
candidates.append(f"/models/{normalized}")
if basename:
candidates.append(f"/models/{basename}")
candidates.extend(alias_map.get(normalized, []))
seen = set()
for candidate in candidates:
if candidate in seen:
continue
seen.add(candidate)
path = Path(candidate)
if (path / "config.json").is_file():
print(f"FOUND_LOCAL_MODEL={candidate}")
raise SystemExit(0)
print("LOCAL_MODEL_NOT_FOUND")
print("Checked candidates:")
for candidate in seen:
print(f" - {candidate}")
raise SystemExit(1)
PY
' > /tmp/local_model_resolution.txt; then
resolved_model_path="$(sed -n 's/^FOUND_LOCAL_MODEL=//p' /tmp/local_model_resolution.txt | tail -n 1)"
if [ -z "${resolved_model_path}" ]; then
echo "ERROR: local model resolver succeeded but did not emit a model path."
cat /tmp/local_model_resolution.txt
exit 1
fi
echo "Resolved local model path: ${resolved_model_path}"
echo "OOT_MODEL_AVAILABLE_LOCALLY=true" >> "$GITHUB_ENV"
echo "OOT_RESOLVED_MODEL_PATH=${resolved_model_path}" >> "$GITHUB_ENV"
else
echo "Local model path not found for ${{ matrix.model_path }}."
cat /tmp/local_model_resolution.txt || true
echo "OOT_MODEL_AVAILABLE_LOCALLY=false" >> "$GITHUB_ENV"
fi
- name: Download model if needed
if: ${{ steps.validation-window.outputs.should_run == 'true' }}
run: |
set -euo pipefail
model_dir="${MODEL_MOUNT_PATH}/${{ matrix.model_path }}"
if [ "${OOT_MODEL_AVAILABLE_LOCALLY:-false}" = "true" ]; then
echo "Using local model path ${OOT_RESOLVED_MODEL_PATH}; skip download."
exit 0
fi
if [ -n "${MODEL_CACHE_MOUNT}" ]; then
lock_suffix="$(printf '%s' "${{ matrix.model_path }}" | tr '/:@' '___')"
if ! $CONTAINER_ENGINE exec \
-e HF_TOKEN="${HF_TOKEN:-}" \
-e MODEL_PATH="${{ matrix.model_path }}" \
-e MODEL_DIR="$model_dir" \
-e RUNNER_NAME="${{ matrix.runner }}" \
-e LOCK_SUFFIX="$lock_suffix" \
"$CONTAINER_NAME" \
bash -lc '
set -euo pipefail
completion_flag="$MODEL_DIR/.download-complete"
use_model_lock="true"
if [ "$RUNNER_NAME" = "atom-mi355-8gpu.predownload" ]; then
use_model_lock="false"
fi
download_model() {
if [ -f "$MODEL_DIR/config.json" ]; then
echo "Model directory exists without a completion marker under ${MODEL_DIR}; validating download"
else
echo "Model not found under ${MODEL_DIR}; downloading model"
fi
rm -f "$completion_flag"
hf download "$MODEL_PATH" --local-dir "$MODEL_DIR"
touch "$completion_flag"
}
if [ -f "$completion_flag" ]; then
echo "Model already exists under ${MODEL_DIR}; skip download"
elif [ "$use_model_lock" != "true" ]; then
echo "Baremetal runner ${RUNNER_NAME} detected; skipping shared model download lock"
download_model
else
lock_dir="/models/.cache/atom-download-locks"
mkdir -p "$lock_dir"
lock_file="$lock_dir/${LOCK_SUFFIX}.lock"
exec 9>"$lock_file"
echo "Waiting for model download lock: $lock_file"
if ! flock -w 7200 9; then
echo "Timed out waiting for model download lock: $lock_file"
exit 1
fi
if [ -f "$completion_flag" ]; then
echo "Model became available under ${MODEL_DIR} while waiting for the lock; skip download"
else
download_model
fi
fi
'; then
echo "Model download failed for '${{ matrix.model_path }}'. Aborting."
exit 1
fi
else
echo "${MODEL_MOUNT_PATH} directory not mounted; skipping model download"
fi
- name: Resolve OOT model path
if: ${{ steps.validation-window.outputs.should_run == 'true' && success() }}
run: |
if [ "${OOT_MODEL_AVAILABLE_LOCALLY:-false}" = "true" ]; then
echo "Using previously resolved local model path: ${OOT_RESOLVED_MODEL_PATH}"
exit 0
fi
if [ -n "${MODEL_CACHE_MOUNT}" ]; then
echo "OOT_RESOLVED_MODEL_PATH=${MODEL_MOUNT_PATH}/${{ matrix.model_path }}" >> "$GITHUB_ENV"
echo "Using mounted model path: ${MODEL_MOUNT_PATH}/${{ matrix.model_path }}"
else
echo "OOT_RESOLVED_MODEL_PATH=${{ matrix.model_path }}" >> "$GITHUB_ENV"
echo "Using model id: ${{ matrix.model_path }}"
fi
- name: Start OOT accuracy server
if: ${{ steps.validation-window.outputs.should_run == 'true' }}
timeout-minutes: 10
env:
OOT_MODEL_NAME: ${{ matrix.model_name }}
OOT_MODEL_PATH: ${{ matrix.model_path }}
OOT_EXTRA_ARGS: ${{ matrix.extra_args }}
run: |
$CONTAINER_ENGINE exec -d \
-e OOT_MODEL_NAME="${OOT_MODEL_NAME}" \
-e OOT_MODEL_PATH="${OOT_RESOLVED_MODEL_PATH:-$OOT_MODEL_PATH}" \
-e OOT_EXTRA_ARGS="${OOT_EXTRA_ARGS}" \
"$CONTAINER_NAME" bash -lc "
set -euo pipefail
bash .github/scripts/atom_oot_test.sh start full
"
- name: Wait for OOT accuracy server
if: ${{ steps.validation-window.outputs.should_run == 'true' && success() }}
timeout-minutes: 120
env:
MAX_WAIT_RETRIES: "120"
WAIT_INTERVAL_SEC: "30"
STREAM_VLLM_LOGS: "1"
run: |
last_vllm_log_line=0
emit_new_vllm_logs() {
if [ "${STREAM_VLLM_LOGS}" != "1" ]; then
return 0
fi
current_line_count=$($CONTAINER_ENGINE exec "$CONTAINER_NAME" bash -lc 'if [ -f /tmp/vllm_oot.log ]; then wc -l < /tmp/vllm_oot.log; else echo 0; fi' 2>/dev/null || echo 0)
current_line_count=${current_line_count//$'\r'/}
if [ "${current_line_count:-0}" -le "${last_vllm_log_line}" ]; then
return 0
fi
echo ""
echo "========== New vLLM log output =========="
$CONTAINER_ENGINE exec "$CONTAINER_NAME" bash -lc "sed -n '$((last_vllm_log_line + 1)),${current_line_count}p' /tmp/vllm_oot.log" || true
last_vllm_log_line=${current_line_count}
}
for ((i=1; i<=MAX_WAIT_RETRIES; i++)); do
if $CONTAINER_ENGINE exec "$CONTAINER_NAME" bash -lc 'curl -fsS http://127.0.0.1:8000/v1/models >/dev/null 2>&1'; then
emit_new_vllm_logs
echo "vLLM server is ready."
exit 0
fi
emit_new_vllm_logs
server_status=$($CONTAINER_ENGINE exec "$CONTAINER_NAME" bash -lc '
if [ -f /tmp/vllm_oot.pid ]; then
pid=$(cat /tmp/vllm_oot.pid)
if kill -0 "$pid" 2>/dev/null; then
echo running
else
echo dead
fi
elif pgrep -f "vllm serve" >/dev/null 2>&1; then
echo running
else
echo starting
fi
' 2>/dev/null || echo unknown)
if [ "${server_status}" = "dead" ]; then
echo "vLLM server process exited before becoming ready."
$CONTAINER_ENGINE exec "$CONTAINER_NAME" bash -lc 'tail -n 200 /tmp/vllm_oot.log || true' || true
exit 1
fi
echo "Waiting for vLLM server... (${i}/${MAX_WAIT_RETRIES}; status=${server_status})"
sleep "${WAIT_INTERVAL_SEC}"
done
echo "vLLM server did not become ready in time."
emit_new_vllm_logs
$CONTAINER_ENGINE exec "$CONTAINER_NAME" bash -lc 'tail -n 200 /tmp/vllm_oot.log || true' || true
exit 1
- name: Run OOT gsm8k accuracy client
id: run_accuracy_client
if: ${{ steps.validation-window.outputs.should_run == 'true' && success() }}
timeout-minutes: 120
continue-on-error: true
env:
OOT_MODEL_NAME: ${{ matrix.model_name }}
OOT_MODEL_PATH: ${{ matrix.model_path }}
OOT_EXTRA_ARGS: ${{ matrix.extra_args }}
OOT_CLIENT_COMMAND: ${{ matrix.client_command || '' }}
LM_EVAL_NUM_FEWSHOT: ${{ matrix.lm_eval_num_fewshot }}
run: |
set -o pipefail
$CONTAINER_ENGINE exec \
-e OOT_MODEL_NAME="${OOT_MODEL_NAME}" \
-e OOT_MODEL_PATH="${OOT_RESOLVED_MODEL_PATH:-$OOT_MODEL_PATH}" \
-e OOT_EXTRA_ARGS="${OOT_EXTRA_ARGS}" \
-e OOT_CLIENT_COMMAND="${OOT_CLIENT_COMMAND}" \
-e OOT_DOCKER_IMAGE="${OOT_IMAGE_TAG}" \
-e GPU_NAME="${{ steps.gpu-info.outputs.gpu_name }}" \
-e GPU_VRAM_GB="${{ steps.gpu-info.outputs.gpu_vram_gb }}" \
-e ROCM_VERSION="${{ steps.gpu-info.outputs.rocm_version }}" \
-e LM_EVAL_NUM_FEWSHOT="${LM_EVAL_NUM_FEWSHOT}" \
"$CONTAINER_NAME" bash -lc "
set -euo pipefail
bash .github/scripts/atom_oot_test.sh client full
" 2>&1 | tee ./oot_accuracy_client.log
- name: Check OOT accuracy test results
if: ${{ steps.validation-window.outputs.should_run == 'true' && success() && steps.run_accuracy_client.outcome == 'success' }}
run: |
$CONTAINER_ENGINE cp "$CONTAINER_NAME":/tmp/oot_accuracy_results ./oot_accuracy_results || true
result_file=$(ls -1t oot_accuracy_results/*.json 2>/dev/null | head -n 1)
if [ -z "$result_file" ] || [ ! -f "$result_file" ]; then
echo "ERROR: No results JSON file found in oot_accuracy_results/"
exit 2
fi
echo "RESULT_FILE: $result_file"
flexible_extract_value=$(jq '.results.gsm8k["exact_match,flexible-extract"]' "$result_file")
echo "Flexible extract value: $flexible_extract_value"
echo "Accuracy test threshold: ${{ matrix.accuracy_test_threshold }}"
result=$(awk -v val="$flexible_extract_value" -v threshold="${{ matrix.accuracy_test_threshold }}" 'BEGIN {print (val < threshold) ? 1 : 0}')
if [ "$result" -eq 1 ]; then
echo "Accuracy test failed: Flexible extract value $flexible_extract_value is less than threshold ${{ matrix.accuracy_test_threshold }}."
exit 1
else
echo "Accuracy test passed: Flexible extract value $flexible_extract_value is greater than or equal to threshold ${{ matrix.accuracy_test_threshold }}."
exit 0
fi
- name: Check OOT MTP acceptance rate
if: ${{ steps.validation-window.outputs.should_run == 'true' && success() && steps.run_accuracy_client.outcome == 'success' && matrix.mtp_accept_threshold != '' }}
env:
MTP_ACCEPT_THRESHOLD: ${{ matrix.mtp_accept_threshold }}
MTP_PER_POS_THRESHOLD: ${{ matrix.mtp_per_pos_threshold }}
run: |
# MTP acceptance is recorded into the result JSON by atom_oot_test.sh
# (scraped from the live vLLM /metrics endpoint during the gsm8k run).
# gsm8k accuracy alone CANNOT guard MTP: speculative decoding is loss-
# less w.r.t. the target model, so a broken draft head leaves accuracy
# unchanged and only craters acceptance/throughput. This step is the
# only gate that catches an MTP regression. Uses jq+awk (no python on
# the host runner), mirroring the gsm8k accuracy check above.
result_file=$(ls -1t oot_accuracy_results/*.json 2>/dev/null | head -n 1)
if [ -z "$result_file" ] || [ ! -f "$result_file" ]; then
echo "ERROR: No results JSON file found for MTP acceptance check."
exit 2
fi
echo "RESULT_FILE: $result_file"
overall=$(jq -r '.atom_ci_metadata.mtp_acceptance_overall // empty' "$result_file")
if [ -z "$overall" ]; then
echo "ERROR: mtp_acceptance_overall missing — spec-decode /metrics were not captured during the run. Treating as a failure."
exit 1
fi
fail=0
echo "MTP overall acceptance: $overall (threshold ${MTP_ACCEPT_THRESHOLD})"
if awk -v v="$overall" -v t="${MTP_ACCEPT_THRESHOLD}" 'BEGIN {exit !(v < t)}'; then
echo "FAIL: overall acceptance $overall < threshold ${MTP_ACCEPT_THRESHOLD}"
fail=1
fi
if [ -n "${MTP_PER_POS_THRESHOLD}" ]; then
IFS=',' read -ra pos_thresholds <<< "${MTP_PER_POS_THRESHOLD}"
idx=0
for th in "${pos_thresholds[@]}"; do
p=$(jq -r ".atom_ci_metadata.mtp_per_pos_acceptance[$idx] // empty" "$result_file")
if [ -z "$p" ]; then
echo "FAIL: position $idx acceptance missing (expected >= $th)"
fail=1
else
echo "MTP position $idx acceptance: $p (threshold $th)"
if awk -v v="$p" -v t="$th" 'BEGIN {exit !(v < t)}'; then
echo "FAIL: position $idx acceptance $p < threshold $th"
fail=1
fi
fi
idx=$((idx + 1))
done
fi
if [ "$fail" -eq 1 ]; then
echo "MTP acceptance gate FAILED."
exit 1
fi
echo "MTP acceptance gate PASSED."
- name: Collect summary
if: ${{ steps.validation-window.outputs.should_run == 'true' && success() }}
run: |
echo "OOT gsm8k summary for ${{ matrix.model_name }}:" >> $GITHUB_STEP_SUMMARY
$CONTAINER_ENGINE exec "$CONTAINER_NAME" bash -lc "awk '/\|Tasks\|Version\|/,/^$/ { if (NF > 0) print }' /tmp/oot_accuracy_output.txt" >> $GITHUB_STEP_SUMMARY || true
- name: Collect artifacts
if: ${{ always() && steps.validation-window.outputs.should_run == 'true' }}
run: |
$CONTAINER_ENGINE cp "$CONTAINER_NAME":/tmp/vllm_oot.log ./vllm_oot.log || true
$CONTAINER_ENGINE cp "$CONTAINER_NAME":/tmp/oot_accuracy_output.txt ./oot_accuracy_output.txt || true
$CONTAINER_ENGINE cp "$CONTAINER_NAME":/tmp/oot_accuracy_results ./oot_accuracy_results || true
if [ ! -f ./oot_accuracy_client.log ]; then : > ./oot_accuracy_client.log; fi
- name: Upload model artifacts
if: ${{ always() && steps.validation-window.outputs.should_run == 'true' }}
uses: actions/upload-artifact@v7
with:
name: oot-validation-${{ matrix.model_name }}-${{ github.run_id }}
path: |
vllm_oot.log
oot_accuracy_client.log
oot_accuracy_output.txt
oot_accuracy_results
- name: Fail if OOT accuracy client failed
if: ${{ always() && steps.validation-window.outputs.should_run == 'true' && steps.run_accuracy_client.outcome == 'failure' }}
run: exit 1
- name: Upload accuracy results for dashboard
if: ${{ always() && steps.validation-window.outputs.should_run == 'true' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.upload_accuracy_to_dashboard)) }}
uses: actions/upload-artifact@v7
with:
name: accuracy-${{ matrix.model_name }}
path: oot_accuracy_results/*.json
if-no-files-found: ignore
- name: Cleanup
if: ${{ always() && steps.validation-window.outputs.should_run == 'true' }}
run: |
$CONTAINER_ENGINE exec "$CONTAINER_NAME" bash -lc "if [ -f /tmp/vllm_oot.pid ]; then kill \$(cat /tmp/vllm_oot.pid) || true; fi" || true
$CONTAINER_ENGINE stop "$CONTAINER_NAME" || true
$CONTAINER_ENGINE rm "$CONTAINER_NAME" || true
$CONTAINER_ENGINE rmi "${OOT_IMAGE_REF:-${OOT_IMAGE_TAG}}" || true
oot-model-accuracy-priority-2:
name: Model Accuracy ${{ matrix.priority }} (${{ matrix.model_name }})
needs: [prepare-oot-image, oot-model-accuracy-priority-1]
if: ${{ always() && needs.prepare-oot-image.result == 'success' && needs.prepare-oot-image.outputs.slot_2_has_models == 'true' }}
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.prepare-oot-image.outputs.model_matrix_slot_2) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 240
permissions:
actions: read
contents: write
env:
CONTAINER_NAME: atom_vllm_oot_validation_${{ matrix.priority }}_${{ strategy.job-index }}
OOT_IMAGE_TAG: ${{ needs.prepare-oot-image.outputs.oot_image_tag }}
steps: *oot-model-accuracy-steps
oot-model-accuracy-priority-3:
name: Model Accuracy ${{ matrix.priority }} (${{ matrix.model_name }})
needs: [prepare-oot-image, oot-model-accuracy-priority-2]
if: ${{ always() && needs.prepare-oot-image.result == 'success' && needs.prepare-oot-image.outputs.slot_3_has_models == 'true' }}
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.prepare-oot-image.outputs.model_matrix_slot_3) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 240
permissions:
actions: read
contents: write
env:
CONTAINER_NAME: atom_vllm_oot_validation_${{ matrix.priority }}_${{ strategy.job-index }}
OOT_IMAGE_TAG: ${{ needs.prepare-oot-image.outputs.oot_image_tag }}
steps: *oot-model-accuracy-steps
# Build the dashboard payload from all uploaded accuracy artifacts after the
# full matrix completes. This keeps dashboard writes off the GPU runners and
# ensures each nightly/manual run publishes at most once.
accuracy-dashboard:
name: Update accuracy dashboard
needs:
- oot-model-accuracy-priority-1
- oot-model-accuracy-priority-2
- oot-model-accuracy-priority-3
if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.upload_accuracy_to_dashboard)) }}
concurrency:
group: gh-pages-deploy
cancel-in-progress: false
runs-on: ubuntu-latest
permissions:
actions: read
contents: write
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: '3.12'
- name: Prepare accuracy artifact directory
run: mkdir -p /tmp/accuracy-results
- name: Download accuracy artifacts
uses: actions/download-artifact@v8
with:
path: /tmp/accuracy-results
pattern: accuracy-*
continue-on-error: true
- name: Count downloaded accuracy artifacts
id: downloaded
run: |
json_count=$(find /tmp/accuracy-results -type f -name '*.json' | wc -l | tr -d ' ')
echo "json_count=${json_count}" >> "$GITHUB_OUTPUT"
echo "Downloaded ${json_count} OOT accuracy JSON files"
- name: List downloaded accuracy artifacts
if: steps.downloaded.outputs.json_count != '0'
run: |
echo "=== Downloaded OOT accuracy artifacts ==="
find /tmp/accuracy-results -type f -name '*.json' | head -20 || echo "No JSON files found"
- name: Transform OOT accuracy results for dashboard
if: steps.downloaded.outputs.json_count != '0'
run: |
python3 .github/scripts/accuracy_to_dashboard.py \
/tmp/accuracy-results \
--output accuracy-benchmark-input.json \
--models .github/benchmark/oot_models_accuracy.json \
--backend "ATOM-vLLM" \
--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
echo "=== Generated entries ==="
cat accuracy-benchmark-input.json
- name: Prepare existing dashboard data
if: steps.downloaded.outputs.json_count != '0'
id: prepare_dashboard_data
uses: ./.github/actions/prepare-dashboard-data
with:
data-path: benchmark-dashboard/data.js
- name: Store OOT accuracy result to dashboard
if: steps.downloaded.outputs.json_count != '0'
uses: benchmark-action/github-action-benchmark@v1
with:
tool: customBiggerIsBetter
output-file-path: accuracy-benchmark-input.json
gh-pages-branch: gh-pages
benchmark-data-dir-path: benchmark-dashboard
auto-push: false
max-items-in-chart: 300
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Validate dashboard data after write
if: steps.downloaded.outputs.json_count != '0'
uses: ./.github/actions/validate-dashboard-data
with:
data-path: benchmark-dashboard/data.js
min-entries: ${{ steps.prepare_dashboard_data.outputs.entry_count }}
- name: Push dashboard data
if: steps.downloaded.outputs.json_count != '0'
run: git push origin gh-pages