Skip to content
Atomesh Benchmark

Atomesh Benchmark #18

Sign in to view logs

Atomesh Benchmark

Atomesh Benchmark #18

Workflow file for this run

.github/workflows/atomesh-benchmark.yaml at d69b063
name: Atomesh Benchmark
on:
push:
branches: [main]
paths:
- 'atom/mesh/**'
- '.github/benchmark/models_atomesh.yaml'
- '.github/scripts/atomesh/**'
- 'atom/entrypoints/openai/**'
- '.github/scripts/slurm_submit_helpers.sh'
- '.github/workflows/atomesh-benchmark.yaml'
pull_request:
branches: [main]
types: [opened, synchronize, reopened, ready_for_review]
paths:
- 'atom/mesh/**'
- '.github/benchmark/models_atomesh.yaml'
- '.github/scripts/atomesh/**'
- 'atom/entrypoints/openai/**'
- '.github/scripts/slurm_submit_helpers.sh'
- '.github/workflows/atomesh-benchmark.yaml'
schedule:
# TW weekly: Saturday 00:00 Beijing time (Friday 16:00 UTC).
- cron: '0 16 * * 5'
# Spur MI350X nightly at 00:10 Beijing time (16:10 UTC).
- cron: '10 16 * * *'
workflow_dispatch:
inputs:
suite:
description: 'Benchmark suite'
required: false
default: nightly
type: choice
options:
- smoke
- nightly
- weekly
- agentic_multi_decode
- accuracy
rdma_smoke_nodes:
description: 'Comma-separated Slurm candidate nodes for the Mooncake RDMA smoke test; Slurm selects two when multiple nodes are listed'
required: false
default: ''
type: string
rdma_smoke_node_count:
description: 'Number of Slurm nodes for the Mooncake RDMA smoke test'
required: false
default: '2'
type: choice
options:
- '1'
- '2'
run_all_models:
description: 'Run all models from models_atomesh.yaml'
required: false
default: false
type: boolean
model_names:
description: 'When run_all_models=false, run all cases in the selected suite for these model families. Comma-separated: DeepSeek-V4-Pro,DeepSeek-V4-Pro-0813,GLM-5.2-MXFP4,Kimi-K3-MXFP4,Kimi-K2.5-MXFP4,MiniMax-M3-MXFP4,MiniMax-M3-MXFP8'
required: false
default: ''
type: string
case_names:
description: |
Overrides model_names. Enter comma-separated case names.
DeepSeek:
ds-v4-1p1d-tp8-smoke,ds-v4-1p1d-tp8,ds-v4-1p1d-dpa-tp8,
ds-v4-1p1d-tp8-mtp3,ds-v4-1p1d-dpa-tp8-mtp1
DeepSeek-V4-Pro-0813 (weekly):
ds-v4-0813-1p1d-dpa-tp8-dspark3-agentic-no-offload-c1,
ds-v4-0813-1p1d-dpa-tp8-dspark3-agentic-no-offload-c2,
ds-v4-0813-1p1d-dpa-tp8-dspark3-agentic-no-offload-c16,
ds-v4-0813-1p1d-dpa-tp8-dspark3-agentic-no-offload-c32,
ds-v4-0813-1p1d-dpa-tp8-dspark3-agentic-no-offload-c128,
ds-v4-0813-1p1d-dpa-tp8-dspark3-agentic-no-offload-c192,
ds-v4-0813-1p1d-dpa-tp8-dspark3-agentic-no-offload-c256
DeepSeek-V4-Pro-0813 (accuracy):
ds-v4-0813-1p1d-tp8-dspark3-gsm8k-only-c16
GLM:
glm-52-mxfp4-1p1d-tp4,glm-52-mxfp4-1p1d-tp8,
glm-52-mxfp4-1p1d-cpp4-dcp4-agentic-lmcache-1m-c48,
glm-52-mxfp4-1p1d-cpp4-dpa4-agentic-lmcache-1m-c36,
glm-52-mxfp4-1p1d-cpp4-dpa4-agentic-lmcache-1m-c40,
glm-52-mxfp4-1p2d-cpp4-dpa4-agentic-lmcache-1m-c48,
glm-52-mxfp4-1p2d-cpp4-dpa4-agentic-lmcache-1m-c56
GLM agentic_multi_decode (1P3D):
glm-52-mxfp4-1p3d-cpp4-tp4-mtp5-agentic-lmcache-1m-c12,
glm-52-mxfp4-1p3d-cpp4-tp4-mtp5-agentic-lmcache-1m-c16,
glm-52-mxfp4-1p3d-cpp4-tp4-mtp5-agentic-lmcache-1m-c24
Kimi-K3:
kimi-k3-mxfp4-1p1d-tp8-dspark7-agentic-1m-c1,
kimi-k3-mxfp4-1p1d-tp8-dspark7-agentic-1m-c2,
kimi-k3-mxfp4-1p1d-tp8-dspark7-agentic-1m-c4,
kimi-k3-mxfp4-1p1d-tp8-dcp8-dspark3-agentic-lmcache-1m-c8,
kimi-k3-mxfp4-1p1d-tp8-dcp8-dspark3-agentic-lmcache-1m-c12,
kimi-k3-mxfp4-1p1d-tp8-dcp8-dspark3-agentic-lmcache-1m-c14,
kimi-k3-mxfp4-1p1d-tp8-dcp8-dspark3-agentic-lmcache-1m-c16,
kimi-k3-mxfp4-1p1d-tp8-dcp8-agentic-lmcache-1m-c32,
kimi-k3-mxfp4-1p1d-tp8-dcp8-agentic-lmcache-1m-c40,
kimi-k3-mxfp4-1p1d-tp8-dcp8-agentic-lmcache-1m-c56,
kimi-k3-mxfp4-1p1d-tp8-dcp8-agentic-lmcache-1m-c64
Kimi-K2.5:
kimi-k25-1p1d-tp4,kimi-k25-1p1d-tp4-tp8,
kimi-k25-2p1d-tp4,kimi-k25-1p2d-tp4
MiniMax:
minimax-m3-fp4-1p1d-tp4,minimax-m3-fp4-2p1d-dpa-tp4,
minimax-m3-fp4-1p1d-tp4-eagle3,minimax-m3-fp4-2p1d-dpa-tp4-eagle3,
minimax-m3-fp8-1p1d-tp4,minimax-m3-fp8-2p1d-dpa-tp4,
minimax-m3-fp8-1p1d-tp4-eagle3,minimax-m3-fp8-2p1d-dpa-tp4-eagle3
required: false
default: ''
type: string
atomesh_image:
description: 'Override ATOM docker image'
required: false
default: rocm/atom-dev:latest
type: string
atomesh_slurm_submit_runner:
description: 'GitHub runner label used to submit Slurm jobs: atomesh-cicd = TW MI355X, atomesh-cicd-mi350 = Spur MI350X, atomesh-cicd-mi355-crusoe = Crusoe MI355X'
required: false
default: atomesh-cicd
type: choice
options:
- atomesh-cicd
- atomesh-cicd-mi350
- atomesh-cicd-mi355-crusoe
atomesh_slurm_account:
description: 'Slurm account: amd-atom uses amd-atom-qos (salloc -A amd-atom --qos=amd-atom-qos); TW: amd-frameworks[42/44/47], amd-tw[36/43]; Crusoe v2 uses amd-aifw-dev'
required: false
default: auto
type: choice
options:
- auto
- amd-atom
- amd-frameworks
- amd-tw
atomesh_slurm_partition:
description: 'TW Slurm partition: amd-frameworks[42/44/47], amd-tw[36/43]; Crusoe v2 uses the default partition'
required: false
default: auto
type: choice
options:
- auto
- amd-frameworks
- amd-tw
- amd-spur
atomesh_single_node:
description: 'Single-node P/D placement; atomesh-cicd auto selects one node from its configured candidate pool'
required: false
default: auto
type: choice
options:
- auto
- pit2-p03-g01
- pit2-p03-g03
- pit2-p03-g07
- pit2-p03-g08
- pit2-p03-g09
- pit2-p03-g10
- pit2-p03-g12
- pit2-p03-g13
- pit2-p03-g15
- pit2-p03-g17
- pit2-p03-g19
- pit2-p03-g20
- pit2-p03-g23
- pit2-p03-g24
- pit2-p03-g25
- pit2-p03-g27
- pit2-p03-g42
atomesh_1p1d_nodes:
description: 'Comma-separated candidate nodes for 1P1D; atomesh-cicd auto uses its configured pool; explicit nodes must belong to that pool'
required: false
default: auto
type: string
atomesh_2p1d_nodes:
description: 'Comma-separated candidate nodes for 2P1D; atomesh-cicd auto uses its configured pool; explicit nodes must belong to that pool'
required: false
default: auto
type: string
benchmark_concurrency:
description: 'Optional benchmark concurrency override, e.g. 1,2,4 (no quotes)'
required: false
default: ''
type: string
eval_concurrency:
description: 'Optional accuracy result-concurrency override; SWE-bench agent workers come from models_atomesh.yaml'
required: false
default: ''
type: string
run_model_benchmark:
description: 'Run model Slurm benchmark for workflow_dispatch'
required: false
default: true
type: boolean
publish_dashboard:
description: 'Publish workflow_dispatch results to the benchmark dashboard'
required: false
default: false
type: boolean
env:
ATOMESH_NODE_POOL: pit2-p03-g01,pit2-p03-g03,pit2-p03-g07,pit2-p03-g08,pit2-p03-g09,pit2-p03-g10,pit2-p03-g12,pit2-p03-g13,pit2-p03-g15,pit2-p03-g17,pit2-p03-g19,pit2-p03-g20,pit2-p03-g23,pit2-p03-g24,pit2-p03-g25,pit2-p03-g27,pit2-p03-g42
# GID 1 selects IPv4 on TW; GID 0 is IPv6 link-local.
NCCL_IB_GID_INDEX: '1'
ATOMESH_SINGLE_NODE: ${{ github.event_name == 'workflow_dispatch' && inputs.atomesh_single_node || 'auto' }}
ATOMESH_SLURM_ACCOUNT: ${{ github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_account != 'auto' && inputs.atomesh_slurm_account || '' }}
ATOMESH_SLURM_PARTITION: ${{ github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_partition != 'auto' && inputs.atomesh_slurm_partition || (((github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-crusoe-mi355' || (github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi355-crusoe') && 'amd-spur' || '') }}
ATOMESH_SLURM_SUBMIT_RUNNER: ${{ github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd') }}
ATOMESH_DASHBOARD_HARDWARE: ${{ (github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi350' && 'MI350X' || 'MI355X' }}
ATOMESH_PD_RANK_MAPPING_POLICY: ${{ (github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi350' && 'idx2idx' || 'none' }}
ATOMESH_LOG_ROOT: ${{ ((github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi350' && '/data/${USER}/logs/ATOMESH_LOG/' || (github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi355-crusoe' && '/home/junyyang/ATOMesh_ACTION_RUNNER/ATOMesh_LOG/' || (github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-crusoe-mi355' && '/shared_nfs/ATOMESH_CI/LOG/' || '/it-share/ATOMESH_LOG/') }}
ATOMESH_MODEL_ROOT: ${{ (github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi350' && '/data/models2' || (((github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-crusoe-mi355' || (github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi355-crusoe') && '/shared_nfs/huggingface_models/amd' || '/mnt/models') }}
ATOMESH_1P1D_NODES: ${{ github.event_name == 'workflow_dispatch' && inputs.atomesh_1p1d_nodes != 'auto' && inputs.atomesh_1p1d_nodes || ((github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi350' && 'ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-5,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-6,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-7,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-8,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-13,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-14,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-15,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-16' || '') }}
ATOMESH_2P1D_NODES: ${{ github.event_name == 'workflow_dispatch' && inputs.atomesh_2p1d_nodes != 'auto' && inputs.atomesh_2p1d_nodes || ((github.event_name == 'workflow_dispatch' && inputs.atomesh_slurm_submit_runner || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd')) == 'atomesh-cicd-mi350' && 'ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-5,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-6,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-7,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-8,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-13,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-14,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-15,ml-ai-ubuntu-gpu-mi350x8-2304gb-fabric-16' || '') }}
concurrency:
# Keep manual benchmark dispatches independent so different cases on the same branch can run together.
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || (github.event_name == 'schedule' && github.event.schedule == '10 16 * * *' && 'atomesh-cicd-mi350' || 'atomesh-cicd') }}
cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' && github.ref != 'refs/heads/main' }}
permissions:
actions: read
contents: write
jobs:
load-config:
if: ${{ !github.event.pull_request || github.event.pull_request.draft == false }}
name: Load ATOMesh benchmark config
runs-on: ubuntu-latest
outputs:
matrix_json: ${{ steps.matrix.outputs.matrix_json }}
has_matrix: ${{ steps.matrix.outputs.has_matrix }}
image: ${{ steps.image.outputs.image }}
steps:
- name: Checkout ATOM repo
uses: actions/checkout@v6
- name: Install YAML parser
run: python3 -m pip install pyyaml
- name: Test ATOMesh config and launch scripts
run: python3 -m unittest discover -s tests -p "test_atomesh_*.py"
- name: Find ATOMesh latest nightly image
id: image
env:
INPUT_ATOMESH_IMAGE: ${{ github.event_name == 'workflow_dispatch' && inputs.atomesh_image || '' }}
run: |
set -euo pipefail
python3 - <<'PY'
import json
import os
import urllib.parse
import urllib.request
from datetime import datetime, timedelta, timezone
from pathlib import Path
import yaml
cfg = yaml.safe_load(Path(".github/benchmark/models_atomesh.yaml").read_text(encoding="utf-8")) or {}
default_image = str(cfg.get("backends", {}).get("atom", {}).get("image", "")).strip()
image = os.environ.get("INPUT_ATOMESH_IMAGE", "").strip() or default_image
def docker_tags(prefix: str) -> list[dict]:
url = (
"https://hub.docker.com/v2/repositories/rocm/atom-dev/tags?"
+ urllib.parse.urlencode({"page_size": 100, "name": prefix})
)
tags = []
while url:
with urllib.request.urlopen(url, timeout=30) as response:
data = json.load(response)
tags.extend(
tag for tag in data.get("results", [])
if str(tag.get("name", "")).startswith(prefix)
)
url = data.get("next")
return tags
def latest_nightly_for(day) -> str | None:
prefix = f"nightly_{day:%Y%m%d}"
tags = docker_tags(prefix)
if not tags:
return None
tags.sort(
key=lambda tag: (str(tag.get("last_updated", "")), str(tag.get("name", ""))),
reverse=True,
)
return f"rocm/atom-dev:{tags[0]['name']}"
def docker_tag_digest(tag_name: str) -> str:
url = f"https://hub.docker.com/v2/repositories/rocm/atom-dev/tags/{urllib.parse.quote(tag_name)}"
with urllib.request.urlopen(url, timeout=30) as response:
data = json.load(response)
if data.get("digest"):
return str(data["digest"])
for image_info in data.get("images", []):
if image_info.get("digest"):
return str(image_info["digest"])
return ""
resolved_image = image
if image == "rocm/atom-dev:latest":
beijing_tz = timezone(timedelta(hours=8))
today = datetime.now(beijing_tz).date()
resolved_image = latest_nightly_for(today)
if resolved_image is None:
resolved_image = latest_nightly_for(today - timedelta(days=1))
if resolved_image is None:
print("::warning::Cannot find rocm/atom-dev nightly tag for today or yesterday; falling back to rocm/atom-dev:latest.")
resolved_image = image
image_digest = ""
if resolved_image == "rocm/atom-dev:latest":
image_digest = docker_tag_digest("latest")
if image_digest:
print(f"ATOMesh image digest: {image_digest}")
else:
print("::warning::Cannot resolve digest for rocm/atom-dev:latest.")
print(f"ATOMesh image: {image} -> {resolved_image}")
with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output:
output.write(f"image={resolved_image}\n")
output.write(f"image_digest={image_digest}\n")
step_summary = os.environ.get("GITHUB_STEP_SUMMARY")
if step_summary and image_digest:
with Path(step_summary).open("a", encoding="utf-8") as summary:
summary.write(f"ATOMesh image digest: `{image_digest}`\n")
PY
- name: Build benchmark matrix
id: matrix
env:
SUITE: ${{ (github.event_name == 'schedule' || github.event_name == 'push') && 'nightly' || (github.event_name == 'workflow_dispatch' && inputs.suite || 'smoke') }}
RUN_ALL_MODELS: ${{ github.event_name == 'workflow_dispatch' && inputs.run_all_models || github.event_name != 'workflow_dispatch' }}
MODEL_NAMES: ${{ github.event_name == 'workflow_dispatch' && inputs.model_names || '' }}
CASE_NAMES: ${{ github.event_name == 'workflow_dispatch' && inputs.case_names || '' }}
EVENT_SCHEDULE: ${{ github.event_name == 'schedule' && github.event.schedule || '' }}
ATOMESH_IMAGE: ${{ steps.image.outputs.image }}
ATOMESH_BENCHMARK_CONCURRENCY: ${{ github.event_name == 'workflow_dispatch' && inputs.benchmark_concurrency || '' }}
ATOMESH_EVAL_CONCURRENCY: ${{ github.event_name == 'workflow_dispatch' && inputs.eval_concurrency || '' }}
run: |
set -euo pipefail
MODEL_NAME=""
CASE_NAME=""
if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
if [[ -n "${CASE_NAMES}" ]]; then
CASE_NAME="${CASE_NAMES}"
elif [[ "${RUN_ALL_MODELS}" != "true" ]]; then
if [[ -z "${MODEL_NAMES}" && "${SUITE}" != "smoke" ]]; then
echo "::error::Set run_all_models=true, provide model_names, or provide case_names."
exit 1
fi
MODEL_NAME="${MODEL_NAMES}"
fi
fi
if [[ "${SUITE}" == "smoke" && -z "${CASE_NAME}" && -z "${MODEL_NAME}" && "${RUN_ALL_MODELS}" != "true" ]]; then
python3 - <<'PY'
import json
import os
from pathlib import Path
matrix = {"include": []}
Path("atomesh-matrix.json").write_text(
json.dumps(matrix, indent=2),
encoding="utf-8",
)
with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output:
output.write('matrix_json={"include":[]}\n')
output.write("has_matrix=false\n")
print("RDMA smoke workflow: skipping benchmark matrix generation")
PY
cat atomesh-matrix.json
exit 0
fi
if [[ "${GITHUB_EVENT_NAME}" == "schedule" && "${EVENT_SCHEDULE}" == "0 16 * * 5" ]]; then
ATOMESH_SLURM_ACCOUNT=amd-frameworks \
ATOMESH_SLURM_PARTITION=amd-frameworks \
ATOMESH_SLURM_SUBMIT_RUNNER=atomesh-cicd \
python3 .github/scripts/atomesh/pd_matrix.py \
--suite nightly \
--model DeepSeek-V4-Pro \
${ATOMESH_IMAGE:+--image "$ATOMESH_IMAGE"} \
--output atomesh-matrix-deepseek.json
ATOMESH_SLURM_ACCOUNT=amd-frameworks \
ATOMESH_SLURM_PARTITION=amd-frameworks \
ATOMESH_SLURM_SUBMIT_RUNNER=atomesh-cicd \
python3 .github/scripts/atomesh/pd_matrix.py \
--suite weekly \
--model DeepSeek-V4-Pro-0813 \
${ATOMESH_IMAGE:+--image "$ATOMESH_IMAGE"} \
--output atomesh-matrix-v4-weekly.json
ATOMESH_SLURM_ACCOUNT=amd-tw \
ATOMESH_SLURM_PARTITION=amd-tw \
ATOMESH_SLURM_SUBMIT_RUNNER=atomesh-cicd \
python3 .github/scripts/atomesh/pd_matrix.py \
--suite nightly \
--model GLM-5.2-MXFP4 \
--benchmark-kind aiperf_agentic \
${ATOMESH_IMAGE:+--image "$ATOMESH_IMAGE"} \
--output atomesh-matrix-glm.json
python3 - <<'PY'
import json
import os
from pathlib import Path
deepseek = json.loads(Path("atomesh-matrix-deepseek.json").read_text(encoding="utf-8")).get("include", [])
v4_weekly = json.loads(Path("atomesh-matrix-v4-weekly.json").read_text(encoding="utf-8")).get("include", [])
glm_agentic = json.loads(Path("atomesh-matrix-glm.json").read_text(encoding="utf-8")).get("include", [])
matrix = {"include": deepseek + v4_weekly + glm_agentic}
Path("atomesh-matrix.json").write_text(
json.dumps(matrix, indent=2),
encoding="utf-8",
)
with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output:
output.write(f"matrix_json={json.dumps(matrix, separators=(',', ':'))}\n")
output.write(f"has_matrix={'true' if matrix['include'] else 'false'}\n")
print(
"Scheduled TW weekly matrix: "
f"{len(deepseek)} DeepSeek case(s), {len(v4_weekly)} V4-0813 weekly case(s), "
f"{len(glm_agentic)} GLM agentic case(s)"
)
PY
else
python3 .github/scripts/atomesh/pd_matrix.py \
--suite "$SUITE" \
${MODEL_NAME:+--model "$MODEL_NAME"} \
${CASE_NAME:+--case "$CASE_NAME"} \
${ATOMESH_IMAGE:+--image "$ATOMESH_IMAGE"} \
${ATOMESH_BENCHMARK_CONCURRENCY:+--benchmark-concurrency "$ATOMESH_BENCHMARK_CONCURRENCY"} \
${ATOMESH_EVAL_CONCURRENCY:+--eval-concurrency "$ATOMESH_EVAL_CONCURRENCY"} \
--output atomesh-matrix.json \
--github-output
fi
cat atomesh-matrix.json
- name: Validate ATOMesh P/D matrix
if: ${{ steps.matrix.outputs.has_matrix == 'true' }}
env:
RESULT_DIR: atomesh-dry-run-results
run: |
set -euo pipefail
chmod +x .github/scripts/atomesh/*.sh .github/scripts/atomesh/*.py
python3 - <<'PY'
import json
import os
import subprocess
from pathlib import Path
matrix = json.loads(Path("atomesh-matrix.json").read_text(encoding="utf-8"))
for cell in matrix.get("include", []):
print(f"Validating {cell['id']}", flush=True)
subprocess.run(
[
"bash",
".github/scripts/atomesh/pd_submit.sh",
"--cell-json",
json.dumps(cell, separators=(",", ":")),
"--result-dir",
os.environ["RESULT_DIR"],
"--dry-run",
],
check=True,
)
PY
- name: Upload expanded matrix
uses: actions/upload-artifact@v7
with:
name: atomesh-benchmark-matrix
path: atomesh-matrix.json
rdma-smoke-test:
name: Mooncake RDMA smoke test
needs: [load-config]
if: ${{ github.event_name == 'workflow_dispatch' && inputs.suite == 'smoke' }}
runs-on: ${{ inputs.atomesh_slurm_submit_runner }}
timeout-minutes: 75
env:
MC_GID_INDEX: '1'
RDMA_SMOKE_IMAGE: ${{ needs.load-config.outputs.image }}
RDMA_SMOKE_NODES: ${{ inputs.rdma_smoke_nodes }}
RDMA_SMOKE_NODE_COUNT: ${{ inputs.rdma_smoke_node_count }}
RDMA_SMOKE_DECODE_HCA_MODE: all
SLURM_SUBMIT_RUNNER: ${{ inputs.atomesh_slurm_submit_runner }}
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Run Mooncake RDMA transfer test on selected nodes
id: rdma-smoke
run: |
set -euo pipefail
export PATH="/usr/local/slurm-24.05.5.1/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}"
if [[ -z "${RDMA_SMOKE_IMAGE}" ]]; then
echo "::error::No ATOM image was resolved for the RDMA smoke test."
exit 1
fi
if [[ ! "${MC_GID_INDEX}" =~ ^[0-9]+$ ]]; then
echo "::error::MC_GID_INDEX must be a non-negative integer."
exit 2
fi
nodes="${RDMA_SMOKE_NODES:-${ATOMESH_1P1D_NODES:-}}"
nodes="${nodes// /}"
if [[ "${SLURM_SUBMIT_RUNNER}" == "atomesh-cicd" ]]; then
nodes="${nodes:-${ATOMESH_NODE_POOL}}"
IFS=',' read -r -a requested_nodes <<< "${nodes}"
for node in "${requested_nodes[@]}"; do
if [[ ",${ATOMESH_NODE_POOL}," != *",${node},"* ]]; then
echo "::error::Node ${node} is outside the atomesh-cicd candidate pool."
exit 2
fi
done
fi
RDMA_SMOKE_TARGET_NODE_COUNT="${RDMA_SMOKE_NODE_COUNT:-2}"
if [[ -n "${nodes}" ]]; then
IFS=',' read -r -a node_list <<< "${nodes}"
node_count="${#node_list[@]}"
if [[ "${node_count}" -gt "${RDMA_SMOKE_TARGET_NODE_COUNT}" ]]; then
node_count="${RDMA_SMOKE_TARGET_NODE_COUNT}"
fi
echo "=== requesting ${node_count} node(s) from ${#node_list[@]} Slurm candidates ==="
else
# Allocate two nodes by default so both local and cross-node GDR are validated.
node_count="${RDMA_SMOKE_TARGET_NODE_COUNT}"
fi
if [[ "${node_count}" -lt 1 ]]; then
echo "::error::GPUDirect RDMA smoke testing requires at least one Slurm node."
exit 1
fi
current_user="$(id -un 2>/dev/null || id -u)"
log_root="${ATOMESH_LOG_ROOT//\$\{USER\}/${current_user}}"
log_root="${log_root//\$USER/${current_user}}"
if [[ "$(hostname -s)" == "pit2-vm-amd-xl-02" ]]; then
log_root="/share_nfs/ATOMESH_RUNNER/ATOMESH_LOG"
fi
run_dir="${log_root%/}/rdma-smoke-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
mkdir -p "${run_dir}"
chmod 0777 "${run_dir}"
export RDMA_SMOKE_RUN_DIR="${run_dir}"
export RDMA_SMOKE_NODE_COUNT="${node_count}"
CURRENT_USER="${current_user}"
ATOMESH_CELL_ID="rdma-smoke"
RESULT_DIR="${run_dir}"
SLURM_JOB_NAME="rdma-smoke-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
SLURM_OUTPUT="${run_dir}/slurm-%j.out"
SLURM_ERROR="${run_dir}/slurm-%j.err"
SLURM_CANCEL_HELPER="${run_dir}/slurm-cancel.sh"
SLURM_LOG_POLL_INTERVAL="${SLURM_LOG_POLL_INTERVAL:-15}"
SLURM_ACCOUNT="${ATOMESH_SLURM_ACCOUNT:-}"
SLURM_PARTITION="${ATOMESH_SLURM_PARTITION:-}"
SLURM_QOS=""
USES_SPUR_CONTROLLER=0
SPUR_CONTROLLER_ADDR="${SPUR_CONTROLLER_ADDR:-}"
SPUR_ACCOUNTING_ADDR=""
if [[ "${SLURM_SUBMIT_RUNNER}" == "atomesh-cicd-mi350" ]]; then
USES_SPUR_CONTROLLER=1
SPUR_CONTROLLER_ADDR="http://134.199.196.72:6817"
SPUR_ACCOUNTING_ADDR="http://134.199.196.72:6819"
elif [[ "${SLURM_SUBMIT_RUNNER}" == "atomesh-cicd-mi355-crusoe" ]]; then
USES_SPUR_CONTROLLER=1
SPUR_CONTROLLER_ADDR="http://crs-m2m-cpu-spur-v2-001.crusoe.amd.com:6817"
SPUR_ACCOUNTING_ADDR="${SPUR_V2_ACCOUNTING_ADDR:-http://crs-m2m-cpu-spur-v2-001.crusoe.amd.com:6819}"
SLURM_ACCOUNT="amd-aifw-dev"
SLURM_PARTITION=""
SLURM_QOS="amd-aifw-dev-qos"
fi
if [[ "$(hostname -s)" == "pit2-vm-amd-xl-02" ]]; then
SLURM_PARTITION="amd-spur"
SLURM_QOS="amd-frameworks-qos"
fi
if [[ "${SLURM_ACCOUNT}" == "amd-atom" ]]; then
SLURM_QOS="amd-atom-qos"
fi
source "${GITHUB_WORKSPACE}/.github/scripts/slurm_submit_helpers.sh"
detect_slurm_backend
install_slurm_cancel_traps
rank_script="${run_dir}/run-rdma-smoke-rank.sh"
cat > "${rank_script}" <<'RANK_SCRIPT'
#!/usr/bin/env bash
set -euo pipefail
rank="${SLURM_PROCID:-${SPUR_TASK_OFFSET:-0}}"
job_id="${SLURM_JOB_ID:-${SPUR_JOB_ID:-local}}"
container="atomesh-rdma-smoke-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${job_id}-${rank}-$$"
rank_host="$(hostname)"
cleanup() {
echo "=== Removing RDMA smoke container ${container} ==="
# Do not wait on D-state RDMA leftovers: kill/rm with a bound so
# the Slurm task can exit instead of sitting in COMPLETING.
if command -v timeout >/dev/null 2>&1; then
timeout 20 docker kill "${container}" >/dev/null 2>&1 || true
timeout 20 docker rm -f "${container}" >/dev/null 2>&1 || true
else
docker kill "${container}" >/dev/null 2>&1 || true
docker rm -f "${container}" >/dev/null 2>&1 || true
fi
}
on_error() {
local rc="$1"
local line="$2"
local command="$3"
echo "ERROR: RDMA smoke rank ${rank} failed on ${rank_host}: rc=${rc} line=${line} command=${command}" >&2
}
on_signal() {
local signal="$1"
local rc="$2"
echo "ERROR: RDMA smoke rank ${rank} received ${signal} on ${rank_host}" >&2
exit "${rc}"
}
publish_rank_rc() {
local rc="$1"
local rc_file
[[ -n "${RDMA_SMOKE_RUN_DIR:-}" ]] || return 0
rc_file="${RDMA_SMOKE_RUN_DIR}/rank-rc-${rank}"
printf '%s\n' "${rc}" > "${rc_file}.tmp" 2>/dev/null || return 0
mv "${rc_file}.tmp" "${rc_file}" 2>/dev/null || true
}
on_exit() {
local rc=$?
trap - EXIT
echo "=== RDMA smoke rank exit: rank=${rank} host=${rank_host} pid=$$ job=${job_id} container=${container} rc=${rc} ==="
# Publish before cleanup: Slurm accounting is unreliable on Spur, so
# the workflow falls back to these per-rank codes.
publish_rank_rc "${rc}"
cleanup
exit "${rc}"
}
trap 'on_error "$?" "${LINENO}" "${BASH_COMMAND}"' ERR
trap on_exit EXIT
trap 'on_signal HUP 129' HUP
trap 'on_signal INT 130' INT
trap 'on_signal TERM 143' TERM
cleanup
echo "=== RDMA smoke rank start: rank=${rank} host=${rank_host} pid=$$ job=${job_id} container=${container} ==="
echo "=== Pulling RDMA smoke image on rank ${rank}: ${RDMA_SMOKE_IMAGE} ==="
docker pull "${RDMA_SMOKE_IMAGE}"
echo "=== RDMA smoke image pull complete on rank ${rank} ==="
docker_args=(
run -i --name "${container}"
--network host --ipc host
--privileged
--device=/dev/kfd
--device=/dev/dri
--device=/dev/infiniband
--group-add=video
--cap-add=IPC_LOCK
--ulimit memlock=-1:-1
-e MC_GID_INDEX="${MC_GID_INDEX:-1}"
-e MC_ENABLE_DEST_DEVICE_AFFINITY=1
-e SMOKE_RANK="${rank}"
-e SMOKE_NODE_COUNT="${RDMA_SMOKE_NODE_COUNT}"
-e SMOKE_DECODE_HCA_MODE="${RDMA_SMOKE_DECODE_HCA_MODE}"
-e SMOKE_RUN_DIR=/rdma-smoke-shared
-v "${RDMA_SMOKE_RUN_DIR}:/rdma-smoke-shared"
)
host_ionic="$(readlink -f /usr/lib/x86_64-linux-gnu/libionic.so.1 2>/dev/null || true)"
[[ -z "${host_ionic}" ]] || docker_args+=(
-v "${host_ionic}:/usr/lib/x86_64-linux-gnu/libionic.so.1:ro"
)
[[ ! -e /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so ]] || docker_args+=(
-v /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav34.so:ro
)
[[ ! -e /etc/libibverbs.d/ionic.driver ]] || docker_args+=(
-v /etc/libibverbs.d/ionic.driver:/etc/libibverbs.d/ionic.driver:ro
)
docker_args+=(
"${RDMA_SMOKE_IMAGE}"
bash -s
)
printf '=== RDMA smoke docker command on rank %s ===' "${rank}"
printf ' %q' docker "${docker_args[@]}"
echo
echo "=== Starting RDMA smoke container on rank ${rank} ==="
set +e
docker "${docker_args[@]}" <<'CONTAINER_SCRIPT'
set -euo pipefail
mapfile -t rdma_devices < <(python3 - <<'PY'
import os
import re
devices = [
name
for name in os.listdir("/sys/class/infiniband")
if os.path.isdir(os.path.join("/sys/class/infiniband", name))
]
def sort_key(name: str) -> int:
match = re.search(r"(?:ionic_|rdma)(\d+)$", name)
return int(match.group(1)) if match else 10_000
for device in sorted(devices, key=sort_key):
print(device)
PY
)
if [[ "${#rdma_devices[@]}" -eq 0 ]]; then
echo "ERROR: no RDMA devices found under /sys/class/infiniband" >&2
exit 1
fi
echo "RDMA devices: ${rdma_devices[*]}"
echo "Mooncake RDMA GID index: ${MC_GID_INDEX}"
for rdma_device in "${rdma_devices[@]}"; do
gid_path="/sys/class/infiniband/${rdma_device}/ports/1/gids/${MC_GID_INDEX}"
selected_gid="$(cat "${gid_path}")"
echo "${rdma_device}/1 GID[${MC_GID_INDEX}]: ${selected_gid}"
done
ibv_devinfo
rocm-smi --showproductname || true
gpu_count="$(python3 - <<'PY'
import torch
print(torch.cuda.device_count())
PY
)"
if ! [[ "${gpu_count}" =~ ^[1-9][0-9]*$ ]]; then
echo "ERROR: no ROCm GPUs are visible for GPUDirect RDMA testing" >&2
exit 1
fi
echo "Visible ROCm GPUs: ${gpu_count}"
validator_source="/app/mooncake/mooncake-transfer-engine/example/transfer_engine_validator.cpp"
python3 - "${validator_source}" <<'PY'
import pathlib
import sys
path = pathlib.Path(sys.argv[1])
source = path.read_text()
replacements = {
""" fillData(thread_id, addr, seed);
submitRequestSync(engine, segment_id, thread_id, addr, remote_base,
TransferRequest::WRITE);
fillData(thread_id, addr, 0);
submitRequestSync(engine, segment_id, thread_id, addr, remote_base,
TransferRequest::READ);
checkData(thread_id, addr, seed);""":
""" fillData(thread_id, addr, seed);
submitRequestSync(engine, segment_id, thread_id, addr, remote_base,
TransferRequest::WRITE);""",
""" submitRequestSync(engine, segment_id, thread_id, addr, remote_base,
TransferRequest::WRITE);
submitRequestSync(engine, segment_id, thread_id, addr, remote_base,
TransferRequest::READ);""":
""" submitRequestSync(engine, segment_id, thread_id, addr, remote_base,
TransferRequest::WRITE);""",
'LOG(INFO) << "Worker " << thread_id << " stopped! Data validation passed";':
'LOG(INFO) << "Worker " << thread_id << " stopped! RDMA WRITE transfers completed";',
}
for old, new in replacements.items():
if source.count(old) != 1:
raise RuntimeError(
f"Expected exactly one validator WRITE/READ sequence, "
f"found {source.count(old)} in {path}"
)
source = source.replace(old, new)
path.write_text(source)
print(f"Patched {path} to exercise production WRITE-only transfers")
PY
build_dir="/tmp/mooncake-rdma-smoke-build"
rm -rf "${build_dir}"
cmake -S /app/mooncake -B "${build_dir}" \
-DUSE_HIP=ON \
-DUSE_ETCD=OFF \
-DBUILD_UNIT_TESTS=ON
cmake --build "${build_dir}" \
--target transfer_engine_validator \
--parallel "$(nproc)"
validator_bin="${build_dir}/mooncake-transfer-engine/example/transfer_engine_validator"
if [[ ! -x "${validator_bin}" ]]; then
echo "ERROR: transfer_engine_validator was not built at ${validator_bin}" >&2
exit 1
fi
rdma_device_csv="$(IFS=,; echo "${rdma_devices[*]}")"
export RDMA_DEVICE_CSV="${rdma_device_csv}"
export GPU_COUNT="${gpu_count}"
python3 - <<'PY'
import json
import os
from pathlib import Path
devices = os.environ["RDMA_DEVICE_CSV"].split(",")
gpu_count = int(os.environ["GPU_COUNT"])
matrix = {}
for gpu_id in range(gpu_count):
matrix[f"cuda:{gpu_id}"] = [devices, []]
Path("/tmp/rdma-smoke-nic-priority.json").write_text(
json.dumps(matrix),
encoding="utf-8",
)
PY
echo "NIC priority matrix:"
cat /tmp/rdma-smoke-nic-priority.json
target_pid=""
log_progress() {
local msg="$*"
echo "${msg}"
echo "$(date -Is) rank=${SMOKE_RANK:-?} ${msg}" >> "${SMOKE_RUN_DIR}/rank-${SMOKE_RANK:-unknown}.log" 2>/dev/null || true
}
start_target_process() {
local gpu_id="$1"
local log_file="$2"
shift 2
if command -v setsid >/dev/null 2>&1; then
HIP_VISIBLE_DEVICES="${gpu_id}" \
setsid "${validator_bin}" "$@" >"${log_file}" 2>&1 < /dev/null &
else
HIP_VISIBLE_DEVICES="${gpu_id}" \
"${validator_bin}" "$@" >"${log_file}" 2>&1 < /dev/null &
fi
target_pid=$!
}
stop_target_process() {
local pid="${target_pid:-}"
[[ -n "${pid}" ]] || return 0
target_pid=""
# SIGTERM runs Mooncake unregisterLocalMemory(), which can hang in
# D-state. Smoke targets are ephemeral: KILL the process group and
# do not wait, so the next GPU pair can start immediately.
echo "WARNING: killing RDMA target process group ${pid} without waiting" >&2
kill -KILL -- "-${pid}" 2>/dev/null || kill -KILL "${pid}" 2>/dev/null || true
disown "${pid}" 2>/dev/null || true
}
cleanup_processes() {
stop_target_process
pkill -KILL -f 'transfer_engine_validator' 2>/dev/null || true
}
trap cleanup_processes EXIT HUP INT TERM
wait_for_p2p_address() {
local target_log="$1"
local target_pid="$2"
local timeout_seconds="$3"
local deadline=$((SECONDS + timeout_seconds))
local address=""
while true; do
address="$(awk '
/Transfer Engine RPC using P2P handshake, listening on / {
print $NF
exit
}
' "${target_log}" 2>/dev/null || true)"
if [[ -n "${address}" ]] && kill -0 "${target_pid}" 2>/dev/null; then
printf '%s\n' "${address}"
return 0
fi
if ! kill -0 "${target_pid}" 2>/dev/null; then
echo "ERROR: P2P target exited before publishing its listening address" >&2
cat "${target_log}" >&2 || true
return 1
fi
if [[ "${SECONDS}" -ge "${deadline}" ]]; then
echo "ERROR: timed out waiting for P2P target listening address" >&2
cat "${target_log}" >&2 || true
return 1
fi
sleep 1
done
}
if [[ "${gpu_count}" -ne 8 ]]; then
echo "ERROR: node-local prefill/decode RDMA testing requires exactly 8 GPUs, found ${gpu_count}" >&2
exit 1
fi
if [[ "${#rdma_devices[@]}" -lt 8 ]]; then
echo "ERROR: node-local prefill/decode RDMA testing requires at least 8 RDMA devices, found ${#rdma_devices[@]}" >&2
exit 1
fi
HCA_COUNT=8
select_all_ib_devices_for_gpu() {
local phys_idx="$1"
local primary="${rdma_devices[phys_idx]}"
local -a selected=("${primary}")
local idx device existing
for ((idx = 0; idx < HCA_COUNT; idx++)); do
device="${rdma_devices[idx]:-}"
[[ -z "${device}" ]] && continue
for existing in "${selected[@]}"; do
[[ "${existing}" == "${device}" ]] && continue 2
done
selected+=("${device}")
done
(IFS=,; echo "${selected[*]}")
}
select_decode_ib_devices() {
local prefill_idx="$1"
local decode_idx="$2"
local prefill_device="${rdma_devices[prefill_idx]}"
local decode_device="${rdma_devices[decode_idx]}"
case "${SMOKE_DECODE_HCA_MODE}" in
local|same-gpu)
echo "${decode_device}"
;;
peer)
echo "${prefill_device}"
;;
local-peer)
if [[ "${decode_device}" == "${prefill_device}" ]]; then
echo "${decode_device}"
else
echo "${decode_device},${prefill_device}"
fi
;;
all)
select_all_ib_devices_for_gpu "${decode_idx}"
;;
*)
echo "ERROR: unsupported SMOKE_DECODE_HCA_MODE=${SMOKE_DECODE_HCA_MODE}" >&2
return 1
;;
esac
}
expected_decode_peer_device() {
local prefill_idx="$1"
local decode_idx="$2"
if [[ "${SMOKE_DECODE_HCA_MODE}" == "local" ||
"${SMOKE_DECODE_HCA_MODE}" == "same-gpu" ]]; then
echo "${rdma_devices[decode_idx]}"
else
echo "${rdma_devices[prefill_idx]}"
fi
}
write_cuda0_nic_matrix() {
local matrix_path="$1"
shift
MATRIX_PATH="${matrix_path}" RDMA_MATRIX_DEVICES="${*}" python3 - <<'PY'
import json
import os
from pathlib import Path
devices = os.environ["RDMA_MATRIX_DEVICES"].split()
Path(os.environ["MATRIX_PATH"]).write_text(
json.dumps({"cuda:0": [devices, []]}),
encoding="utf-8",
)
PY
}
echo "Decode HCA diagnostic mode: ${SMOKE_DECODE_HCA_MODE}"
local_prefill_gpu_ids=()
local_decode_gpu_ids=()
if [[ "${SMOKE_DECODE_HCA_MODE}" == "same-gpu" ]]; then
for ((gpu_id = 0; gpu_id < gpu_count; gpu_id++)); do
local_prefill_gpu_ids+=("${gpu_id}")
local_decode_gpu_ids+=("${gpu_id}")
done
else
for ((prefill_gpu_id = 0; prefill_gpu_id < gpu_count - 1; prefill_gpu_id++)); do
for ((decode_gpu_id = prefill_gpu_id + 1; decode_gpu_id < gpu_count; decode_gpu_id++)); do
local_prefill_gpu_ids+=("${prefill_gpu_id}")
local_decode_gpu_ids+=("${decode_gpu_id}")
done
done
fi
for ((pair_index = 0; pair_index < ${#local_prefill_gpu_ids[@]}; pair_index++)); do
prefill_gpu_id="${local_prefill_gpu_ids[pair_index]}"
decode_gpu_id="${local_decode_gpu_ids[pair_index]}"
prefill_rdma_device="${rdma_devices[prefill_gpu_id]}"
decode_rdma_device="${rdma_devices[decode_gpu_id]}"
decode_device_csv="$(select_decode_ib_devices "${prefill_gpu_id}" "${decode_gpu_id}")"
prefill_matrix="/tmp/gdr-smoke-prefill-gpu-${prefill_gpu_id}-nic-priority.json"
decode_matrix="/tmp/gdr-smoke-decode-gpu-${decode_gpu_id}-nic-priority.json"
write_cuda0_nic_matrix "${prefill_matrix}" "${prefill_rdma_device}"
IFS=',' read -r -a decode_ib_devices <<< "${decode_device_csv}"
write_cuda0_nic_matrix "${decode_matrix}" "${decode_ib_devices[@]}"
target_log="/tmp/gdr-smoke-decode-gpu-${decode_gpu_id}-target.log"
initiator_log="/tmp/gdr-smoke-prefill-gpu-${prefill_gpu_id}-initiator.log"
echo "=== GPUDirect RDMA prefill GPU ${prefill_gpu_id} (${prefill_rdma_device}) -> decode GPU ${decode_gpu_id} (${decode_device_csv}) ==="
start_target_process "${decode_gpu_id}" "${target_log}" \
--mode=target \
--protocol=rdma \
--metadata_server=P2PHANDSHAKE \
--local_server_name=127.0.0.2:0 \
--device_name="${decode_device_csv}" \
--nic_priority_matrix="${decode_matrix}" \
--use_vram=true \
--gpu_id=0 \
--buffer_size=100663296
target_name="$(wait_for_p2p_address "${target_log}" "${target_pid}" 30)"
echo "P2P target listening at ${target_name}"
set +e
MC_LOG_RDMA_SLICE_AFFINITY=1 \
GLOG_v=1 \
HIP_VISIBLE_DEVICES="${prefill_gpu_id}" \
timeout 120 "${validator_bin}" \
--mode=initiator \
--protocol=rdma \
--metadata_server=P2PHANDSHAKE \
--local_server_name=127.0.0.3:0 \
--segment_id="${target_name}" \
--device_name="${prefill_rdma_device}" \
--nic_priority_matrix="${prefill_matrix}" \
--use_vram=true \
--gpu_id=0 \
--buffer_size=100663296 \
--batch_size=1 \
--block_size=67108864 \
--threads=1 \
--duration=1 \
2>&1 | tee "${initiator_log}"
transfer_rc="${PIPESTATUS[0]}"
set -e
if [[ "${transfer_rc}" -ne 0 ]] ||
! grep -q "RDMA WRITE transfers completed" "${initiator_log}"; then
cat "${initiator_log}" >&2
cat "${target_log}" >&2
echo "ERROR: GPUDirect RDMA WRITE failed from prefill GPU ${prefill_gpu_id} (${prefill_rdma_device}) to decode GPU ${decode_gpu_id} (${decode_device_csv})" >&2
exit 1
fi
expected_peer_device="$(expected_decode_peer_device "${prefill_gpu_id}" "${decode_gpu_id}")"
expected_affinity="local_device_name=${prefill_rdma_device}, peer_device_name=${expected_peer_device}"
if grep -Fq "RDMA slice affinity:" "${initiator_log}"; then
if ! grep -Fq "${expected_affinity}" "${initiator_log}"; then
grep -F "RDMA slice affinity:" "${initiator_log}" >&2 || true
echo "ERROR: GPUDirect RDMA selected the wrong HCA for prefill GPU ${prefill_gpu_id}" >&2
exit 1
fi
else
echo "WARNING: RDMA WRITE completed, but HCA affinity logging was unavailable for prefill GPU ${prefill_gpu_id}" >&2
fi
stop_target_process
touch "${SMOKE_RUN_DIR}/node-local-passed-${SMOKE_RANK}-${prefill_gpu_id}-${decode_gpu_id}"
echo "GPUDirect RDMA passed from prefill GPU ${prefill_gpu_id} to decode GPU ${decode_gpu_id} via decode HCAs ${decode_device_csv}"
done
echo "Node-local GPUDirect RDMA passed for all ${#local_prefill_gpu_ids[@]} prefill/decode GPU pairs"
if [[ "${SMOKE_NODE_COUNT}" -lt 2 ]]; then
echo "Single-node GPUDirect RDMA smoke test passed"
exit 0
fi
touch "${SMOKE_RUN_DIR}/local-gdr-done-${SMOKE_RANK}"
if [[ "${SMOKE_RANK}" -le 1 ]]; then
deadline=$((SECONDS + 300))
while true; do
completed_nodes=0
for marker in "${SMOKE_RUN_DIR}"/local-gdr-done-*; do
[[ -e "${marker}" ]] && completed_nodes=$((completed_nodes + 1))
done
[[ "${completed_nodes}" -ge "${SMOKE_NODE_COUNT}" ]] && break
if [[ "${SECONDS}" -ge "${deadline}" ]]; then
echo "ERROR: timed out waiting for node-local GDR tests" >&2
exit 1
fi
sleep 2
done
fi
wait_for_file() {
local path="$1"
local timeout_seconds="$2"
local description="$3"
local deadline=$((SECONDS + timeout_seconds))
while [[ ! -s "${path}" ]]; do
if [[ "${SECONDS}" -ge "${deadline}" ]]; then
echo "ERROR: timed out waiting for ${description}: ${path}" >&2
exit 1
fi
sleep 1
done
}
local_ip=""
if command -v ip >/dev/null 2>&1; then
local_ip="$(ip route get 1.1.1.1 2>/dev/null | awk '
{
for (i = 1; i <= NF; i++) {
if ($i == "src" && i < NF) {
print $(i + 1)
exit
}
}
}
' || true)"
fi
if [[ -z "${local_ip}" ]]; then
local_ip="$(hostname -I 2>/dev/null | awk '{print $1}' || true)"
fi
if [[ -z "${local_ip}" ]]; then
echo "ERROR: unable to determine node IP for cross-node GDR testing" >&2
exit 1
fi
peer_info="${SMOKE_RUN_DIR}/cross-node-prefill-info"
if [[ "${SMOKE_RANK}" -eq 0 ]]; then
{
echo "${local_ip}"
echo "${gpu_count}"
} > "${peer_info}.tmp"
mv "${peer_info}.tmp" "${peer_info}"
elif [[ "${SMOKE_RANK}" -eq 1 ]]; then
wait_for_file "${peer_info}" 60 "prefill node information"
mapfile -t peer_data < "${peer_info}"
prefill_node_ip="${peer_data[0]:-}"
prefill_gpu_count="${peer_data[1]:-0}"
if [[ -z "${prefill_node_ip}" || "${prefill_gpu_count}" -ne "${gpu_count}" ]]; then
echo "ERROR: cross-node GPU topology mismatch: prefill=${prefill_gpu_count}, decode=${gpu_count}" >&2
exit 1
fi
fi
if [[ "${SMOKE_RANK}" -eq 0 || "${SMOKE_RANK}" -eq 1 ]]; then
cross_prefill_gpu_ids=()
cross_decode_gpu_ids=()
for ((gpu_id = 0; gpu_id < gpu_count; gpu_id++)); do
cross_prefill_gpu_ids+=("${gpu_id}")
cross_decode_gpu_ids+=("${gpu_id}")
done
for ((prefill_gpu_id = 0; prefill_gpu_id < gpu_count - 1; prefill_gpu_id++)); do
for ((decode_gpu_id = prefill_gpu_id + 1; decode_gpu_id < gpu_count; decode_gpu_id++)); do
cross_prefill_gpu_ids+=("${prefill_gpu_id}")
cross_decode_gpu_ids+=("${decode_gpu_id}")
done
done
for ((pair_index = 0; pair_index < ${#cross_prefill_gpu_ids[@]}; pair_index++)); do
prefill_gpu_id="${cross_prefill_gpu_ids[pair_index]}"
decode_gpu_id="${cross_decode_gpu_ids[pair_index]}"
prefill_rdma_device="${rdma_devices[prefill_gpu_id]}"
decode_rdma_device="${rdma_devices[decode_gpu_id]}"
decode_device_csv="${decode_rdma_device}"
expected_peer_device="${decode_rdma_device}"
is_cross_rail=0
if [[ "${prefill_gpu_id}" -ne "${decode_gpu_id}" ]]; then
decode_device_csv="$(select_decode_ib_devices "${prefill_gpu_id}" "${decode_gpu_id}")"
expected_peer_device="$(expected_decode_peer_device "${prefill_gpu_id}" "${decode_gpu_id}")"
is_cross_rail=1
fi
echo "=== Cross-node RDMA pair: prefill GPU ${prefill_gpu_id} (${prefill_rdma_device}) -> decode GPU ${decode_gpu_id} (${decode_device_csv}); expected peer HCA ${expected_peer_device} ==="
prefill_matrix="/tmp/cross-node-prefill-gpu-${prefill_gpu_id}-nic-priority.json"
decode_matrix="/tmp/cross-node-decode-gpu-${decode_gpu_id}-nic-priority.json"
write_cuda0_nic_matrix "${prefill_matrix}" "${prefill_rdma_device}"
if [[ "${is_cross_rail}" -eq 1 ]]; then
IFS=',' read -r -a decode_ib_devices <<< "${decode_device_csv}"
write_cuda0_nic_matrix "${decode_matrix}" "${decode_ib_devices[@]}"
else
write_cuda0_nic_matrix "${decode_matrix}" "${decode_rdma_device}"
fi
ready_file="${SMOKE_RUN_DIR}/cross-node-ready-prefill-${prefill_gpu_id}-decode-${decode_gpu_id}"
result_file="${SMOKE_RUN_DIR}/cross-node-result-prefill-${prefill_gpu_id}-decode-${decode_gpu_id}"
target_log="${SMOKE_RUN_DIR}/cross-node-prefill-${prefill_gpu_id}-decode-${decode_gpu_id}-target.log"
if [[ "${SMOKE_RANK}" -eq 1 ]]; then
rm -f \
"${ready_file}" "${ready_file}.tmp" \
"${result_file}" "${result_file}.tmp"
log_progress "=== Cross-node decode target: ${local_ip}, GPU ${decode_gpu_id} (${decode_rdma_device}) ==="
start_target_process "${decode_gpu_id}" "${target_log}" \
--mode=target \
--protocol=rdma \
--metadata_server=P2PHANDSHAKE \
--local_server_name="${local_ip}:0" \
--device_name="${decode_device_csv}" \
--nic_priority_matrix="${decode_matrix}" \
--use_vram=true \
--gpu_id=0 \
--buffer_size=100663296
target_name="$(wait_for_p2p_address "${target_log}" "${target_pid}" 30)"
log_progress "Cross-node P2P target listening at ${target_name}"
echo "${target_name}" > "${ready_file}.tmp"
mv "${ready_file}.tmp" "${ready_file}"
wait_for_file "${result_file}" 150 "prefill GPU ${prefill_gpu_id} to decode GPU ${decode_gpu_id} result"
result="$(<"${result_file}")"
stop_target_process
if [[ "${result}" != "0" ]]; then
cat "${target_log}" >&2
echo "ERROR: cross-node GPUDirect RDMA failed from prefill GPU ${prefill_gpu_id} to decode GPU ${decode_gpu_id}" >&2
exit 1
fi
log_progress "Cross-node GPUDirect RDMA passed on decode GPU ${decode_gpu_id}"
else
wait_for_file "${ready_file}" 120 "decode GPU ${decode_gpu_id} target"
target_name="$(<"${ready_file}")"
initiator_log="/tmp/cross-node-prefill-${prefill_gpu_id}-decode-${decode_gpu_id}-initiator.log"
echo "=== Cross-node prefill initiator: ${local_ip} GPU ${prefill_gpu_id} (${prefill_rdma_device}) -> ${target_name} decode GPU ${decode_gpu_id} ==="
transfer_batch_size=4
transfer_block_size=1048576
transfer_duration=2
affinity_env=()
if [[ "${is_cross_rail}" -eq 1 ]]; then
transfer_batch_size=1
transfer_block_size=67108864
transfer_duration=1
affinity_env=(MC_LOG_RDMA_SLICE_AFFINITY=1 GLOG_v=1)
fi
set +e
env "${affinity_env[@]}" HIP_VISIBLE_DEVICES="${prefill_gpu_id}" \
timeout 120 "${validator_bin}" \
--mode=initiator \
--protocol=rdma \
--metadata_server=P2PHANDSHAKE \
--local_server_name="${local_ip}:0" \
--segment_id="${target_name}" \
--device_name="${prefill_rdma_device}" \
--nic_priority_matrix="${prefill_matrix}" \
--use_vram=true \
--gpu_id=0 \
--buffer_size=100663296 \
--batch_size="${transfer_batch_size}" \
--block_size="${transfer_block_size}" \
--threads=1 \
--duration="${transfer_duration}" \
2>&1 | tee "${initiator_log}"
transfer_rc="${PIPESTATUS[0]}"
set -e
if [[ "${transfer_rc}" -eq 0 ]] &&
grep -q "RDMA WRITE transfers completed" "${initiator_log}"; then
result=0
else
result=1
fi
if [[ "${result}" -eq 0 && "${is_cross_rail}" -eq 1 ]]; then
expected_affinity="local_device_name=${prefill_rdma_device}, peer_device_name=${expected_peer_device}"
if grep -Fq "RDMA slice affinity:" "${initiator_log}"; then
if ! grep -Fq "${expected_affinity}" "${initiator_log}"; then
grep -F "RDMA slice affinity:" "${initiator_log}" >&2 || true
echo "ERROR: cross-node GPUDirect RDMA selected the wrong HCA from prefill GPU ${prefill_gpu_id} to decode GPU ${decode_gpu_id}" >&2
result=1
fi
else
echo "WARNING: cross-node RDMA WRITE completed, but HCA affinity logging was unavailable from prefill GPU ${prefill_gpu_id} to decode GPU ${decode_gpu_id}" >&2
fi
fi
echo "${result}" > "${result_file}.tmp"
mv "${result_file}.tmp" "${result_file}"
if [[ "${result}" -ne 0 ]]; then
cat "${target_log}" >&2 || true
echo "ERROR: cross-node GPUDirect RDMA WRITE failed from prefill GPU ${prefill_gpu_id} to decode GPU ${decode_gpu_id}" >&2
exit 1
fi
touch "${SMOKE_RUN_DIR}/cross-node-passed-${prefill_gpu_id}-${decode_gpu_id}"
echo "Cross-node GPUDirect RDMA passed from prefill GPU ${prefill_gpu_id} to decode GPU ${decode_gpu_id}"
fi
done
fi
if [[ "${SMOKE_RANK}" -eq 0 ]]; then
echo "Mooncake cross-node GPUDirect RDMA passed for same-index GPU pairs 0-7 and all cross-rail pairs i < j"
fi
cleanup_processes
log_progress "rank ${SMOKE_RANK} container script complete"
CONTAINER_SCRIPT
docker_rc=$?
set -e
echo "=== RDMA smoke container finished on rank ${rank}: rc=${docker_rc} ==="
cleanup
exit "${docker_rc}"
RANK_SCRIPT
chmod +x "${rank_script}"
slurm_status_file="${run_dir}/slurm-job.rc"
rm -f "${slurm_status_file}" "${slurm_status_file}.tmp"
rm -f \
"${run_dir}"/rank-rc-* \
"${run_dir}"/node-local-passed-* \
"${run_dir}"/cross-node-passed-*
batch_script="${run_dir}/rdma-smoke.sbatch.sh"
if [[ "${USES_SPUR_CONTROLLER}" == "1" ]]; then
# Spur launches this script on each node with SPUR_TASK_OFFSET; srun is unavailable.
cat > "${batch_script}" <<EOF
#!/usr/bin/env bash
set -euo pipefail
exec "${rank_script}"
EOF
else
cat > "${batch_script}" <<EOF
#!/usr/bin/env bash
set -euo pipefail
set +e
srun --nodes=${node_count} --ntasks=${node_count} --ntasks-per-node=1 \
"${rank_script}"
srun_rc=\$?
set -e
printf '%s\n' "\${srun_rc}" > "${slurm_status_file}.tmp"
mv "${slurm_status_file}.tmp" "${slurm_status_file}"
exit "\${srun_rc}"
EOF
fi
chmod +x "${batch_script}"
sbatch_cmd=(
sbatch
--parsable
--exclusive
--export=ALL
--job-name "${SLURM_JOB_NAME}"
--nodes "${node_count}"
--ntasks "${node_count}"
--ntasks-per-node 1
--time 01:00:00
--output "${SLURM_OUTPUT}"
--error "${SLURM_ERROR}"
)
if [[ -n "${nodes}" ]]; then
slurm_node_selection_args "${nodes}" "${node_count}"
sbatch_cmd+=("${SLURM_NODE_SELECTION_ARGS[@]}")
fi
if [[ "${SLURM_SUBMIT_RUNNER}" == "atomesh-cicd-mi350" ]]; then
sbatch_cmd+=(--controller "${SPUR_CONTROLLER_ADDR}")
else
[[ -z "${SLURM_ACCOUNT}" ]] || sbatch_cmd+=(--account "${SLURM_ACCOUNT}")
[[ -z "${SLURM_PARTITION}" ]] || sbatch_cmd+=(--partition "${SLURM_PARTITION}")
[[ -z "${SLURM_QOS}" ]] || sbatch_cmd+=(--qos "${SLURM_QOS}")
sbatch_cmd+=(--gres gpu:8)
if [[ "${USES_SPUR_CONTROLLER}" == "1" && -n "${SPUR_CONTROLLER_ADDR}" ]]; then
sbatch_cmd+=(
--controller "${SPUR_CONTROLLER_ADDR}"
)
fi
fi
sbatch_cmd+=("${batch_script}")
printf ' %q' "${sbatch_cmd[@]}"
echo
write_slurm_cancel_helper ""
set +e
sbatch_output="$("${sbatch_cmd[@]}")"
sbatch_rc=$?
set -e
echo "${sbatch_output}"
if [[ "${sbatch_rc}" -ne 0 ]]; then
echo "sbatch submit exit code: ${sbatch_rc}"
exit "${sbatch_rc}"
fi
JOB_ID="$(parse_sbatch_job_id "${sbatch_output}")"
SLURM_JOB_ACTIVE=1
echo "${JOB_ID}" | tee "${run_dir}/slurm-job-id"
write_slurm_cancel_helper "${JOB_ID}"
set_slurm_job_log_paths "${JOB_ID}"
SLURM_STATUS_DIR="${run_dir}"
SLURM_STATUS_RANKS="${node_count}"
monitor_slurm_job "${JOB_ID}"
# Spur leaves the job in COMPLETING while RDMA leftovers are reaped,
# so give sacct longer than the 30s default to publish a final state.
SLURM_ACCOUNTING_TIMEOUT="${SLURM_ACCOUNTING_TIMEOUT:-180}"
read_slurm_exit_code "${JOB_ID}"
read_slurm_status_files "${run_dir}" "${node_count}"
SLURM_JOB_ACTIVE=0
echo "slurm_state=${SLURM_STATE}"
echo "slurm_exit_code=${SLURM_EXIT_CODE}"
echo "slurm job exit code: ${SLURM_JOB_RC}"
exit "${SLURM_JOB_RC}"
- name: Summarize Mooncake RDMA smoke test
if: always()
env:
RDMA_SMOKE_OUTCOME: ${{ steps.rdma-smoke.outcome }}
run: |
set -uo pipefail
current_user="$(id -un 2>/dev/null || id -u)"
log_root="${ATOMESH_LOG_ROOT//\$\{USER\}/${current_user}}"
log_root="${log_root//\$USER/${current_user}}"
if [[ "$(hostname -s)" == "pit2-vm-amd-xl-02" ]]; then
log_root="/share_nfs/ATOMESH_RUNNER/ATOMESH_LOG"
fi
run_dir="${log_root%/}/rdma-smoke-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
summary_path="${run_dir}/rdma-smoke-summary.md"
mkdir -p "${run_dir}"
if [[ "${RDMA_SMOKE_OUTCOME}" == "success" ]]; then
outcome_icon="✅"
else
outcome_icon="❌"
fi
job_id=""
if [[ -s "${run_dir}/slurm-job-id" ]]; then
job_id="$(tr -d '[:space:]' < "${run_dir}/slurm-job-id")"
fi
shopt -s nullglob
log_files=("${run_dir}"/slurm-*.out "${run_dir}"/slurm-*.err)
shopt -u nullglob
{
echo "## Mooncake RDMA smoke test"
echo
echo "- Workflow result: ${outcome_icon} \`${RDMA_SMOKE_OUTCOME}\`"
echo "- Image: \`${RDMA_SMOKE_IMAGE}\`"
echo "- Requested nodes: \`${RDMA_SMOKE_NODE_COUNT}\`"
echo "- Decode HCA mode: \`${RDMA_SMOKE_DECODE_HCA_MODE}\`"
echo "- Mooncake GID index: \`${MC_GID_INDEX}\`"
echo "- Slurm job: \`${job_id:-not submitted}\`"
echo
echo "### RDMA WRITE results"
python3 - "${run_dir}" <<'PY'
import os
import sys
from pathlib import Path
run_dir = Path(sys.argv[1])
node_local_results = set()
for path in run_dir.glob("node-local-passed-*-*-*"):
try:
_, _, _, rank, src, dst = path.name.split("-")
node_local_results.add((int(rank), int(src), int(dst)))
except ValueError:
continue
cross_node_pairs = set()
for path in run_dir.glob("cross-node-passed-*-*"):
try:
_, _, _, src, dst = path.name.split("-")
cross_node_pairs.add((int(src), int(dst)))
except ValueError:
continue
cross_rail_pairs = {(src, dst) for src in range(7) for dst in range(src + 1, 8)}
same_index_pairs = {(gpu, gpu) for gpu in range(8)}
node_count = int(os.environ.get("RDMA_SMOKE_NODE_COUNT", "2"))
expected_node_local_results = {
(rank, src, dst)
for rank in range(node_count)
for src, dst in cross_rail_pairs
}
pair_label = "GPU" + "、".join(
f"{src}→{dst}" for src, dst in sorted(cross_rail_pairs)
)
node_local_icon = "✅" if expected_node_local_results <= node_local_results else "❌"
same_index_icon = "✅" if same_index_pairs <= cross_node_pairs else "❌"
cross_rail_icon = "✅" if cross_rail_pairs <= cross_node_pairs else "❌"
print(f"Node-local {pair_label} {node_local_icon}")
print(f"Cross-node same-index GPU0–7 {same_index_icon}")
print(f"Cross-node cross-rail {pair_label} {cross_rail_icon}")
PY
echo
echo "### Errors"
if [[ "${#log_files[@]}" -gt 0 ]] &&
awk '
/^(ERROR:|E[0-9]{8})/ ||
/Permission denied|command not found|Detect data integrity/ {
if (!seen[$0]++ && count++ < 20) {
print "- ❌ `" $0 "`"
}
}
END { exit(count ? 0 : 1) }
' "${log_files[@]}"; then
:
else
echo "- No RDMA smoke errors found in Slurm logs."
fi
} | tee "${summary_path}" >> "${GITHUB_STEP_SUMMARY}"
echo "RDMA smoke summary written to ${summary_path}"
run-model-benchmark:
name: ${{ matrix.id }}
needs: [load-config]
if: >-
needs.load-config.outputs.has_matrix == 'true'
&& (
github.event_name == 'schedule'
|| (github.event_name == 'workflow_dispatch' && inputs.run_model_benchmark)
)
runs-on: ${{ matrix.runner.slurm_submit_runner }}
timeout-minutes: ${{ matrix.run_eval && matrix.accuracy.task == 'swebench_lite' && 780 || 420 }}
strategy:
fail-fast: false
max-parallel: 1
matrix: ${{ fromJson(needs.load-config.outputs.matrix_json) }}
steps:
- name: Checkout ATOM repo
uses: actions/checkout@v6
- name: Cancel existing jobs on benchmark nodes
if: ${{ github.event_name == 'schedule' && matrix.runner.slurm_submit_runner != 'atomesh-cicd' }}
env:
NODES: ${{ join(matrix.nodes, ',') }}
SLURM_SUBMIT_RUNNER: ${{ matrix.runner.slurm_submit_runner }}
run: |
set -euo pipefail
export PATH="/usr/local/slurm-24.05.5.1/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}"
if [[ "${SLURM_SUBMIT_RUNNER}" == "atomesh-cicd-mi350" ]]; then
echo "Slurm job cleanup is disabled for ${SLURM_SUBMIT_RUNNER}; skipping."
exit 0
fi
if [[ -z "${NODES}" ]]; then
echo "No explicit benchmark nodes were selected; skipping Slurm job cleanup."
exit 0
fi
for command_name in squeue scancel; do
if ! command -v "${command_name}" >/dev/null 2>&1; then
echo "ERROR: ${command_name} not found on $(hostname). PATH=${PATH}" >&2
exit 127
fi
done
squeue_cmd=(squeue)
scancel_cmd=(scancel)
if [[ "${SLURM_SUBMIT_RUNNER}" == "atomesh-cicd-mi355-crusoe" ]]; then
controller="http://crs-m2m-cpu-spur-v2-001.crusoe.amd.com:6817"
squeue_cmd+=(--controller "${controller}")
scancel_cmd+=(--controller "${controller}")
fi
echo "=== Checking all users' Slurm jobs on ${NODES} ==="
job_ids="$("${squeue_cmd[@]}" --noheader -w "${NODES}" --format="%A" | sort -u)"
if [[ -z "${job_ids}" ]]; then
echo "No existing Slurm jobs found on ${NODES}."
exit 0
fi
echo "Cancelling Slurm jobs on ${NODES}:"
printf '%s\n' "${job_ids}"
mapfile -t job_id_list <<< "${job_ids}"
for job_id in "${job_id_list[@]}"; do
if timeout 60s "${scancel_cmd[@]}" "${job_id}"; then
echo "Finished cancelling Slurm job ${job_id}."
else
status=$?
if [[ "${status}" -eq 124 ]]; then
echo "ERROR: Timed out after 60 seconds cancelling Slurm job ${job_id}." >&2
else
echo "ERROR: Failed cancelling Slurm job ${job_id} with exit code ${status}." >&2
fi
exit "${status}"
fi
done
- name: Resolve machine-specific benchmark paths
id: machine-paths
env:
CELL_JSON: ${{ toJson(matrix) }}
run: |
set -euo pipefail
python3 - <<'PY'
import json
import os
import socket
from pathlib import Path
cell = json.loads(os.environ["CELL_JSON"])
# The matrix is expanded on ubuntu-latest, before the submit machine is known.
machine_name = socket.gethostname().split(".", 1)[0]
if machine_name == "pit2-vm-amd-xl-02":
old_model_root = os.environ["ATOMESH_MODEL_ROOT"].rstrip("/")
def remap_model_paths(value):
if isinstance(value, dict):
return {key: remap_model_paths(item) for key, item in value.items()}
if isinstance(value, list):
return [remap_model_paths(item) for item in value]
if isinstance(value, str) and (value == old_model_root or value.startswith(old_model_root + "/")):
return "/share_nfs/models" + value[len(old_model_root):]
return value
cell = remap_model_paths(cell)
cell["runner"]["log_root"] = "/share_nfs/ATOMESH_RUNNER/ATOMESH_LOG"
cell["runner"]["slurm_partition"] = "amd-spur"
cell["runner"]["slurm_qos"] = "amd-frameworks-qos"
if (
cell["runner"].get("slurm_account") == "amd-atom"
and cell["runner"].get("slurm_submit_runner") != "atomesh-cicd-mi355-crusoe"
):
cell["runner"]["slurm_qos"] = "amd-atom-qos"
with Path(os.environ["GITHUB_OUTPUT"]).open("a", encoding="utf-8") as output:
output.write(f"cell_json={json.dumps(cell, separators=(',', ':'))}\n")
PY
- name: Print benchmark config
env:
CELL_JSON: ${{ steps.machine-paths.outputs.cell_json }}
run: |
set -euo pipefail
python3 - <<'PY'
import json
import os
cell = json.loads(os.environ["CELL_JSON"])
def get(path, default=""):
cur = cell
for key in path.split("."):
if not isinstance(cur, dict) or key not in cur:
return default
cur = cur[key]
return cur
summary = {
"id": get("id"),
"suite": get("suite"),
"model": get("model"),
"case": get("name"),
"image": get("image"),
"model_path": get("model_path"),
"precision": get("precision"),
"runner": get("runner.slurm_submit_runner"),
"slurm_account": get("runner.slurm_account"),
"slurm_partition": get("runner.slurm_partition"),
"slurm_qos": get("runner.slurm_qos"),
"log_root": get("runner.log_root"),
"nodes": ",".join(get("nodes", [])),
"num_nodes": get("num_nodes"),
"topology": get("topology"),
"pd_worker_layout": get("pd_worker_layout"),
"prefill": get("service.prefill"),
"decode": get("service.decode"),
"router": get("service.router"),
"server_args": get("server_args"),
"isl": get("isl"),
"osl": get("osl"),
"concurrency": get("concurrency"),
"run_eval": get("run_eval"),
"accuracy": get("accuracy"),
}
print("=== ATOMesh benchmark config summary ===")
for key, value in summary.items():
print(f"{key}: {value}")
print("=== ATOMesh benchmark full cell JSON ===")
print(json.dumps(cell, indent=2, sort_keys=True))
PY
- name: Run Slurm benchmark cell
env:
# Build the reviewed source in a separate setup container. Use the
# existing release profile for both benchmark and eval services.
ATOMESH_BUILD_MESH: ${{ matrix.benchmark.kind == 'aiperf_agentic' && 'true' || 'false' }}
ATOMESH_MESH_BUILD_PROFILE: release
CELL_JSON: ${{ steps.machine-paths.outputs.cell_json }}
RESULT_DIR: atomesh-results
SLURM_SUBMIT_RUNNER: ${{ matrix.runner.slurm_submit_runner }}
run: |
set -euo pipefail
ORIGINAL_WORKSPACE="${GITHUB_WORKSPACE}"
CURRENT_USER="$(id -un 2>/dev/null || id -u)"
if [[ "${SLURM_SUBMIT_RUNNER}" == "atomesh-cicd-mi350" ]]; then
SHARED_WORKSPACE="/data/${CURRENT_USER}/atomesh-ci/${GITHUB_RUN_ID}/${{ matrix.id }}"
rm -rf "${SHARED_WORKSPACE}"
mkdir -p "${SHARED_WORKSPACE}"
if command -v rsync >/dev/null 2>&1; then
rsync -a --delete "${ORIGINAL_WORKSPACE}/" "${SHARED_WORKSPACE}/"
else
tar -C "${ORIGINAL_WORKSPACE}" -cf - . | tar -C "${SHARED_WORKSPACE}" -xf -
fi
cd "${SHARED_WORKSPACE}"
RESULT_DIR="${ORIGINAL_WORKSPACE}/${RESULT_DIR}"
fi
chmod +x .github/scripts/atomesh/*.sh .github/scripts/atomesh/*.py
SLURM_CANCEL_HELPER="${RESULT_DIR}/${{ matrix.id }}.slurm-cancel.sh"
SLURM_JOB_NAME="${{ matrix.id }}-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}"
SPUR_CONTROLLER_ADDR="${SPUR_CONTROLLER_ADDR:-}"
if [[ "${SLURM_SUBMIT_RUNNER}" == "atomesh-cicd-mi355-crusoe" ]]; then
SPUR_CONTROLLER_ADDR="http://crs-m2m-cpu-spur-v2-001.crusoe.amd.com:6817"
elif [[ -z "${SPUR_CONTROLLER_ADDR}" && "${SLURM_SUBMIT_RUNNER}" == "atomesh-cicd-mi350" ]]; then
SPUR_CONTROLLER_ADDR="http://134.199.196.72:6817"
fi
source "${GITHUB_WORKSPACE}/.github/scripts/slurm_submit_helpers.sh"
detect_slurm_backend
PD_SUBMIT_PID=""
scancel_slurm_by_name_from_wrapper() {
command -v scancel >/dev/null 2>&1 || return 0
local -a scancel_cmd=(scancel)
if [[ "${USES_SPUR_CONTROLLER}" == "1" && -n "${SPUR_CONTROLLER_ADDR}" ]]; then
scancel_cmd+=(--controller "${SPUR_CONTROLLER_ADDR}")
fi
scancel_cmd+=(--user "${CURRENT_USER}" --name "${SLURM_JOB_NAME}")
if command -v timeout >/dev/null 2>&1; then
timeout "${SLURM_SCANCEL_TIMEOUT_SECONDS:-8}" "${scancel_cmd[@]}" || true
else
"${scancel_cmd[@]}" || true
fi
}
cleanup_slurm_from_wrapper() {
local rc=$?
trap - EXIT HUP INT TERM
if [[ "${rc}" -eq 75 && -s "${SLURM_CANCEL_HELPER}.status-unknown" ]]; then
# Shared helper's EX_TEMPFAIL: status is unknown, not a batch failure.
echo "WARNING: Slurm monitoring failed; preserving job ${SLURM_JOB_NAME}. Check the recorded job ID and controller before retrying."
elif [[ "${rc}" -ne 0 ]]; then
echo "=== cancelling Slurm job from workflow wrapper rc=${rc} ==="
if [[ -x "${SLURM_CANCEL_HELPER}" ]]; then
SLURM_CANCEL_WAIT_SECONDS="${SLURM_CANCEL_WAIT_SECONDS:-5}" \
SLURM_CANCEL_KILL_WAIT_SECONDS="${SLURM_CANCEL_KILL_WAIT_SECONDS:-5}" \
bash "${SLURM_CANCEL_HELPER}" || true
else
echo "Slurm cancel helper is not available yet"
fi
scancel_slurm_by_name_from_wrapper
fi
if [[ -n "${PD_SUBMIT_PID}" ]] && kill -0 "${PD_SUBMIT_PID}" 2>/dev/null; then
kill "${PD_SUBMIT_PID}" 2>/dev/null || true
wait "${PD_SUBMIT_PID}" 2>/dev/null || true
fi
exit "${rc}"
}
trap cleanup_slurm_from_wrapper EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
bash .github/scripts/atomesh/pd_submit.sh \
--cell-json "$CELL_JSON" \
--result-dir "$RESULT_DIR" &
PD_SUBMIT_PID=$!
wait "${PD_SUBMIT_PID}"
- name: Summarize ${{ matrix.id }} benchmark result
if: always()
env:
MATRIX_ID: ${{ matrix.id }}
run: |
set -euo pipefail
mkdir -p atomesh-results
SUMMARY_PATH="atomesh-results/${MATRIX_ID}-summary.md"
AIPERF_RAW_PATH="atomesh-results/${MATRIX_ID}-aiperf-raw.md"
BENCHMARK_ACTION_INPUT="atomesh-results/${MATRIX_ID}-benchmark-action.json"
python3 .github/scripts/atomesh/process_result.py \
"atomesh-results/${MATRIX_ID}" \
--output "${BENCHMARK_ACTION_INPUT}" \
--summary "${SUMMARY_PATH}" \
--hardware "${ATOMESH_DASHBOARD_HARDWARE}" \
--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
# Empty for standard ISL/OSL cases: only AIPerf writes the console dump.
python3 .github/scripts/atomesh/aiperf_console_report.py \
"atomesh-results/${MATRIX_ID}" \
--output "${AIPERF_RAW_PATH}"
{
echo "## ${MATRIX_ID}"
echo
if [ -f "atomesh-results/${MATRIX_ID}.slurm-job-id" ]; then
echo "Slurm job: $(cat "atomesh-results/${MATRIX_ID}.slurm-job-id")"
echo
fi
if [ -f "atomesh-results/${MATRIX_ID}.log-root" ]; then
echo "Log path: \`$(cat "atomesh-results/${MATRIX_ID}.log-root")\`"
echo
fi
cat "${SUMMARY_PATH}"
if [ -s "${AIPERF_RAW_PATH}" ]; then
echo
cat "${AIPERF_RAW_PATH}"
fi
} >> "$GITHUB_STEP_SUMMARY"
cat "${SUMMARY_PATH}"
if [ -s "${AIPERF_RAW_PATH}" ]; then
cat "${AIPERF_RAW_PATH}"
fi
- name: Upload AIPerf Perfetto traces
if: ${{ always() && matrix.benchmark.kind == 'aiperf_agentic' }}
uses: ./.github/actions/upload-aiperf-traces
with:
result-dir: atomesh-results/${{ matrix.id }}
artifact-name: aiperf-chrome-traces-${{ matrix.id }}
output-dir: aiperf-traces/${{ matrix.id }}
run-url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
- name: Prepare agentic PD latency reports
if: ${{ always() && matrix.benchmark.kind == 'aiperf_agentic' }}
id: metrics-reports
env:
MATRIX_ID: ${{ matrix.id }}
run: |
python3 .github/scripts/atomesh/observability/stage_reports.py \
--cell "$MATRIX_ID" \
--output "atomesh-reports/$MATRIX_ID"
- name: Upload agentic PD latency reports
if: ${{ always() && steps.metrics-reports.outcome == 'success' }}
id: metrics-artifact
uses: actions/upload-artifact@v7
with:
name: atomesh-latency-${{ matrix.id }}-${{ github.run_attempt }}
path: atomesh-reports/${{ matrix.id }}/
if-no-files-found: error
- name: Link latency reports in job summary
if: ${{ always() && steps.metrics-artifact.outcome == 'success' }}
env:
MATRIX_ID: ${{ matrix.id }}
METRICS_ARTIFACT_URL: ${{ steps.metrics-artifact.outputs.artifact-url }}
run: |
python3 .github/scripts/atomesh/observability/stage_reports.py \
--cell "$MATRIX_ID" \
--output "atomesh-reports/$MATRIX_ID" \
--summary-only
- name: Upload model benchmark artifacts
if: always()
uses: actions/upload-artifact@v7
with:
name: atomesh-model-benchmark-${{ matrix.id }}
path: |
atomesh-results/
if-no-files-found: ignore
summarize-results:
name: Summarize ATOMesh benchmark results
needs: [run-model-benchmark]
if: ${{ always() && !cancelled() && needs.run-model-benchmark.result != 'skipped' }}
runs-on: ubuntu-latest
outputs:
has_results: ${{ steps.summary.outputs.has_results }}
steps:
- name: Checkout ATOM repo
uses: actions/checkout@v6
- name: Download model benchmark artifacts
uses: actions/download-artifact@v8
with:
pattern: atomesh-model-benchmark-*
merge-multiple: true
path: atomesh-results
- name: Build benchmark summary
id: summary
run: |
set -euo pipefail
mkdir -p atomesh-results
python3 .github/scripts/atomesh/process_result.py \
atomesh-results \
--output atomesh-benchmark-action.json \
--summary atomesh-benchmark-summary.md \
--hardware "${ATOMESH_DASHBOARD_HARDWARE}" \
--run-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
# Empty for standard ISL/OSL cases: only AIPerf writes the console dump.
python3 .github/scripts/atomesh/aiperf_console_report.py \
atomesh-results \
--output atomesh-aiperf-raw.md
python3 - <<'PY' > atomesh-case-log-paths.md
import json
import re
from pathlib import Path
results = Path("atomesh-results")
def sort_key(case_id):
"""Order this table like the perf summary: (benchmark_model_name,
max_concurrency), read from the case's own result JSON so both tables
key off the same fields. A cell that sweeps several concurrencies
leads at its lowest one; a case that died before writing any result
falls back to its id, whose prefix ahead of the NpMd topology token is
the model name.
"""
keys = []
for path in sorted((results / case_id).rglob("pd-*.json")):
try:
payload = json.loads(path.read_text(encoding="utf-8"))
model = str(payload.get("benchmark_model_name") or "")
conc = int(payload.get("max_concurrency") or 0)
except (OSError, ValueError, TypeError):
continue
keys.append((model.rstrip("/").split("/")[-1].lower(), conc))
if keys:
return min(keys)
topology = re.search(r"-\d+p\d+d-", case_id)
found = re.findall(r"-c(\d+)-", case_id)
return (
case_id[: topology.start()] if topology else case_id,
int(found[-1]) if found else 0,
)
rows = []
for path in results.glob("*.log-root"):
case_id = path.name[: -len(".log-root")]
job_file = path.with_name(f"{case_id}.slurm-job-id")
job_id = job_file.read_text().strip() if job_file.is_file() else ""
rows.append((case_id, job_id or "--", path.read_text().strip()))
if rows:
print("### ATOMesh Case Log Paths")
print()
print("| Case | Slurm job | Log path |")
print("| --- | --- | --- |")
for case_id, job_id, log_root in sorted(
rows, key=lambda row: sort_key(row[0]) + (row[0],)
):
print(f"| {case_id} | {job_id} | `{log_root}` |")
print()
PY
if [[ -s atomesh-case-log-paths.md ]]; then
cat atomesh-case-log-paths.md >> "$GITHUB_STEP_SUMMARY"
fi
cat atomesh-benchmark-summary.md >> "$GITHUB_STEP_SUMMARY"
if [[ -s atomesh-aiperf-raw.md ]]; then
{
echo
cat atomesh-aiperf-raw.md
} >> "$GITHUB_STEP_SUMMARY"
fi
ENTRY_COUNT="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("atomesh-benchmark-action.json")
entries = json.loads(path.read_text(encoding="utf-8")) if path.is_file() else []
print(len(entries))
PY
)"
echo "entry_count=${ENTRY_COUNT}" >> "$GITHUB_OUTPUT"
if [[ "${ENTRY_COUNT}" -gt 0 ]]; then
echo "has_results=true" >> "$GITHUB_OUTPUT"
else
echo "has_results=false" >> "$GITHUB_OUTPUT"
echo "No successful ATOMesh benchmark result JSONs were produced; dashboard publish will be skipped." >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload AIPerf Perfetto traces
if: always()
uses: ./.github/actions/upload-aiperf-traces
with:
result-dir: atomesh-results
artifact-name: aiperf-chrome-traces-${{ github.run_id }}
output-dir: aiperf-traces
run-url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
- name: Upload benchmark summary
if: ${{ steps.summary.outputs.has_results == 'true' }}
uses: actions/upload-artifact@v7
with:
name: atomesh-benchmark-summary
path: |
atomesh-benchmark-action.json
atomesh-benchmark-summary.md
atomesh-aiperf-raw.md
atomesh-case-log-paths.md
if-no-files-found: ignore
dashboard:
concurrency:
group: gh-pages-deploy
cancel-in-progress: false
name: Update ATOMesh Benchmark Dashboard
needs: [summarize-results]
if: >-
!cancelled()
&& needs.summarize-results.result == 'success'
&& needs.summarize-results.outputs.has_results == 'true'
&& (
github.event_name == 'schedule'
|| (github.event_name == 'workflow_dispatch' && inputs.publish_dashboard)
)
runs-on: ubuntu-latest
steps:
- name: Checkout ATOM repo
uses: actions/checkout@v6
- name: Download CI benchmark dashboard inputs
uses: actions/download-artifact@v8
with:
name: atomesh-benchmark-summary
path: atomesh-dashboard-input
- name: Prepare ATOMesh dashboard assets
run: |
set -euo pipefail
test -s atomesh-dashboard-input/atomesh-benchmark-action.json
cp .github/dashboard/index.html /tmp/atomesh_dashboard_index.html
- name: Prepare existing dashboard data
id: prepare_dashboard_data
uses: ./.github/actions/prepare-dashboard-data
with:
data-path: benchmark-dashboard/data.js
- name: Store ATOMesh benchmark result to dashboard
uses: benchmark-action/github-action-benchmark@v1
with:
tool: customBiggerIsBetter
output-file-path: atomesh-dashboard-input/atomesh-benchmark-action.json
gh-pages-branch: gh-pages
benchmark-data-dir-path: benchmark-dashboard
auto-push: false
max-items-in-chart: 300
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Validate dashboard data after write
uses: ./.github/actions/validate-dashboard-data
with:
data-path: benchmark-dashboard/data.js
min-entries: ${{ steps.prepare_dashboard_data.outputs.entry_count }}
- name: Deploy ATOMesh dashboard shell to gh-pages
run: |
set -euo pipefail
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git fetch origin gh-pages
git checkout gh-pages
git pull --ff-only origin gh-pages
mkdir -p benchmark-dashboard
cp /tmp/atomesh_dashboard_index.html benchmark-dashboard/index.html
git add benchmark-dashboard/index.html
git diff --cached --quiet || git commit -m "Update ATOMesh benchmark dashboard"
git push origin gh-pages