Repository navigation
SpeedBench AL Collection #144
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: SpeedBench AL Collection | |
| # Push-button (workflow_dispatch) collection of a SPEED-Bench acceptance-length | |
| # (AL) matrix: thinking_on/off x MTP levels, for the given model (defaults to | |
| # DeepSeek-V4-Pro). Produces the golden reference consumed by the | |
| # synthetic-acceptance framework and (optionally) opens a PR updating | |
| # benchmarks/speedbench-reference-al.yaml. | |
| # Independent collection requests use priority-scheduled runner slots. | |
| on: # zizmor: ignore[concurrency-limits] | |
| workflow_dispatch: | |
| inputs: | |
| runner: | |
| description: "Self-hosted GPU runner label (B300)" | |
| required: false | |
| type: string | |
| default: 'b300' | |
| model: | |
| description: "HF model id (basename must be in launcher STAGED_MODELS for pre-staged local weights)" | |
| required: false | |
| type: string | |
| default: 'deepseek-ai/DeepSeek-V4-Pro' | |
| model-prefix: | |
| description: "Model prefix; drives srt-slurm recipe lookup, exp name, and artifact names" | |
| required: false | |
| type: string | |
| default: 'dsv4' | |
| image: | |
| description: "vLLM container image" | |
| required: false | |
| type: string | |
| default: 'vllm/vllm-openai:v0.21.0' | |
| mtp-list: | |
| description: "Space-separated MTP levels (num_speculative_tokens)" | |
| required: false | |
| type: string | |
| default: '1 2 3 4 5 6 7 8' | |
| thinking-modes: | |
| description: "Space-separated thinking modes to collect" | |
| required: false | |
| type: string | |
| default: 'off on' | |
| category: | |
| description: "SPEED-Bench category" | |
| required: false | |
| type: string | |
| default: 'coding' | |
| output-len: | |
| description: "Per-request output length" | |
| required: false | |
| type: string | |
| default: '4096' | |
| thinking-kwargs: | |
| description: "chat_template_kwargs JSON for thinking-on cells (match golden config)" | |
| required: false | |
| type: string | |
| default: '{"thinking": true, "reasoning_effort": "high"}' | |
| salloc-time: | |
| description: "Slurm allocation minutes (16 server starts ~ several hours)" | |
| required: false | |
| type: string | |
| default: '480' | |
| open-pr: | |
| description: "Open a PR updating benchmarks/speedbench-reference-al.yaml (default off: artifact-only, paste values in manually)" | |
| required: false | |
| type: boolean | |
| default: false | |
| ref: | |
| description: "Git ref (branch/sha) to checkout" | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| env: | |
| DRAFT_SAMPLE_METHOD: greedy | |
| PORT: '8888' | |
| INFMAX_CONTAINER_WORKSPACE: '/workspace' | |
| DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1' | |
| MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600' | |
| ENROOT_IMPORT_TIME_LIMIT: '120' | |
| GPU_MONITOR_INTERVAL: '1' | |
| GPU_METRICS_CSV: 'gpu_metrics.csv' | |
| KEEP_LOGS: '0' | |
| IS_MULTINODE: 'false' | |
| EVAL_ONLY: 'false' | |
| RUN_EVAL: 'false' | |
| IS_AGENTIC: '0' | |
| REQUIRE_POWER: '0' | |
| SCENARIO_SUBDIR: fixed_seq_len/ | |
| HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} | |
| HF_HUB_CACHE: '/mnt/hf_hub_cache/' | |
| # Drive the single-node path in runners/launch_b300-dsxe.sh. MODEL is the HF id; | |
| # its basename (e.g. DeepSeek-V4-Pro) must be in the launcher's STAGED_MODELS so | |
| # the launcher resolves MODEL_PATH to the pre-staged local weights and mounts | |
| # them. The collector serves from MODEL_PATH (see SERVE_MODEL), so no download. | |
| MODEL: ${{ inputs.model }} | |
| MODEL_PREFIX: ${{ inputs.model-prefix }} | |
| PRECISION: fp4 | |
| FRAMEWORK: vllm | |
| EXP_NAME: ${{ inputs.model-prefix }}_speedbench | |
| IMAGE: ${{ inputs.image }} | |
| TP: '8' | |
| EP_SIZE: '1' | |
| DP_ATTENTION: 'false' | |
| SPEC_DECODING: mtp | |
| SALLOC_TIME_LIMIT: ${{ inputs.salloc-time }} | |
| # Matrix-collector tunables (propagated into the container via srun --export=ALL). | |
| MTP_LIST: ${{ inputs.mtp-list }} | |
| THINKING_MODES: ${{ inputs.thinking-modes }} | |
| CATEGORY: ${{ inputs.category }} | |
| SPEEDBENCH_OUTPUT_LEN: ${{ inputs.output-len }} | |
| CHAT_TEMPLATE_KWARGS_ON: ${{ inputs.thinking-kwargs }} | |
| OUT_YAML: /workspace/speedbench-reference-al.yaml | |
| PYTHONDONTWRITEBYTECODE: '1' | |
| PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache | |
| jobs: | |
| setup: | |
| name: setup | |
| runs-on: ubuntu-latest | |
| outputs: | |
| priority: ${{ steps.score.outputs.priority }} | |
| queue-token: ${{ steps.score.outputs.queue-token }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: { clean: true, persist-credentials: false } | |
| - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| - id: score | |
| env: | |
| RUNNER: ${{ inputs.runner }} | |
| MODEL_PREFIX: ${{ inputs.model-prefix }} | |
| run: | | |
| entry=$(jq -cn \ | |
| --arg runner "$RUNNER" \ | |
| --arg model_prefix "$MODEL_PREFIX" \ | |
| '[{ | |
| runner: $runner, | |
| framework: "vllm", | |
| "model-prefix": $model_prefix, | |
| precision: "fp4", | |
| "spec-decoding": "mtp" | |
| }]') | |
| scored=$(printf '%s' "$entry" | | |
| uv run --no-project --exclude-newer PT12H --python 3.12 --with pyyaml \ | |
| python -m infx.workflows.ci_priority \ | |
| --queue-namespace "${GITHUB_RUN_ID}:${GITHUB_RUN_ATTEMPT}") | |
| echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT" | |
| echo "queue-token=$(jq -r '.[0]["queue-token"]' <<<"$scored")" >> "$GITHUB_OUTPUT" | |
| collect-al: | |
| needs: setup | |
| runs-on: >- | |
| ${{ fromJSON( | |
| vars.PRIORITY_SCHEDULER_ENABLED == 'true' && | |
| ( | |
| vars.NODE_SLOT_SCHEDULER_ENABLED == 'true' && | |
| format( | |
| '["self-hosted",{0},{1},{2},{3}]', | |
| toJSON(inputs.runner), | |
| '"nodes:1"', | |
| toJSON(format( | |
| 'ci-job-{0}-{1}', | |
| needs.setup.outputs.priority, | |
| needs.setup.outputs.queue-token | |
| )), | |
| toJSON(format('ci-attempt-{0}', github.run_attempt)) | |
| ) || | |
| format( | |
| '["self-hosted",{0},{1},{2}]', | |
| toJSON(inputs.runner), | |
| toJSON(format( | |
| 'ci-job-{0}-{1}', | |
| needs.setup.outputs.priority, | |
| needs.setup.outputs.queue-token | |
| )), | |
| toJSON(format('ci-attempt-{0}', github.run_attempt)) | |
| ) | |
| ) || | |
| format('[{0}]', toJSON(inputs.runner)) | |
| ) }} | |
| timeout-minutes: 600 | |
| name: "p${{ needs.setup.outputs.priority }} | SpeedBench AL matrix | ${{ inputs.category }} | mtp=[${{ inputs.mtp-list }}] | thinking=[${{ inputs.thinking-modes }}]" | |
| steps: | |
| - name: Resource cleanup (pre-run) | |
| run: &resource-cleanup | | |
| # Cleanup Docker resources | |
| if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then | |
| echo "[Docker] Cleaning up resources ..." | |
| docker ps -aq | xargs -r docker rm -f | |
| docker network prune -f | |
| while [ -n "$(docker ps -aq)" ]; do | |
| docker ps -a | |
| sleep 5 | |
| done | |
| fi | |
| # Cleanup SLURM resources | |
| if command -v squeue >/dev/null 2>&1; then | |
| echo "[Slurm] Cleaning up jobs with name: ${RUNNER_NAME} ..." | |
| scancel --name="${RUNNER_NAME}" || true | |
| while [ -n "$(squeue --name="${RUNNER_NAME}" --noheader --format='%i')" ]; do | |
| squeue --name="${RUNNER_NAME}" | |
| sleep 5 | |
| done | |
| fi | |
| # Cleanup AL-matrix outputs from a prior job on this runner so a stale | |
| # matrix from a previous run is never picked up as this job's output. | |
| rm -rf "${GITHUB_WORKSPACE}/speedbench_results" 2>/dev/null || true | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.ref || github.sha }} | |
| clean: true | |
| submodules: true | |
| persist-credentials: false | |
| - name: Cleanup stale outputs (pre-run) | |
| run: | | |
| rm -f speedbench-reference-al.yaml || true | |
| rm -f gpu_metrics.csv || true | |
| rm -rf speed_bench_data || true | |
| - name: Collect AL matrix | |
| env: | |
| RUNNER_NAME: ${{ runner.name }} | |
| run: | | |
| set -eo pipefail | |
| export GPU_COUNT="$TP" | |
| # Historical measured revisions own their configuration in the launchers. | |
| if [[ -f benchmarks/runtime_settings.sh ]]; then | |
| source benchmarks/runtime_settings.sh | |
| fi | |
| if [[ -f runners/runtime_settings.sh ]]; then | |
| source runners/runtime_settings.sh | |
| fi | |
| # All collectors use the srt-slurm single-node path with per-model | |
| # recipes under srt-slurm-recipes/<prefix>/vllm/b300-fp4-speedbench/. | |
| RECIPE_PATH="benchmarks/single_node/srt-slurm-recipes/${MODEL_PREFIX}/vllm/b300-fp4-speedbench/speedbench.yaml" | |
| if [[ ! -f "$RECIPE_PATH" ]]; then | |
| echo "ERROR: SPEED-Bench recipe not found: $RECIPE_PATH" >&2 | |
| exit 1 | |
| fi | |
| # Plain TP8 is incompatible with the Qwen3.8-Flash-Next FP8 checkpoint's | |
| # 128-wide quantization blocks; its recipe runs TP4 (as the legacy | |
| # collector did), so bind the matrix TP and GPU count to match. | |
| if [[ "$MODEL_PREFIX" == qwen3.8next ]]; then | |
| export TP=4 GPU_COUNT=4 | |
| fi | |
| mkdir -p speedbench_results | |
| ALL_FAILED=true | |
| # Harmless values for launch_srt_single_node env checks; the collector | |
| # recipe and client do not read them. | |
| export ISL=256 | |
| export OSL=256 | |
| export RANDOM_RANGE_RATIO=0.0 | |
| export CONC=1 | |
| export PP_SIZE=1 | |
| export DCP_SIZE=1 | |
| export PCP_SIZE=1 | |
| for mode in $THINKING_MODES; do | |
| for mtp in $MTP_LIST; do | |
| IDX=$((mtp - 1)) | |
| export THINKING="$mode" | |
| export MTP="$mtp" | |
| export SRT_RECIPE="${RECIPE_PATH}:zip_override_mtp[${IDX}]" | |
| export RESULT_FILENAME="speedbench_${mode}_mtp${mtp}" | |
| echo "" | |
| echo "==========================================" | |
| echo " srt-slurm cell: thinking=${mode} MTP=${mtp}" | |
| echo " SRT_RECIPE=${SRT_RECIPE}" | |
| echo "==========================================" | |
| CELL_RC=0 | |
| bash ./runners/launch_"${RUNNER_NAME%%_*}".sh || CELL_RC=$? | |
| # Collect per-cell artifacts into the results directory. | |
| if [[ -f "${RESULT_FILENAME}.json" ]]; then | |
| mv "${RESULT_FILENAME}.json" "speedbench_results/${RESULT_FILENAME}.json" | |
| ALL_FAILED=false | |
| fi | |
| if [[ -f srt-single-node-logs.tar.gz ]]; then | |
| mv srt-single-node-logs.tar.gz "speedbench_results/srt-logs_${mode}_mtp${mtp}.tar.gz" | |
| fi | |
| if [[ "$CELL_RC" -ne 0 ]]; then | |
| echo " -> cell failed (rc=$CELL_RC), recording N/A" | |
| fi | |
| done | |
| done | |
| if [[ "$ALL_FAILED" == true ]]; then | |
| echo "ERROR: every SPEED-Bench cell failed" >&2 | |
| exit 1 | |
| fi | |
| # Aggregate per-cell result JSONs into the reference YAML. | |
| PYTHONPATH="${GITHUB_WORKSPACE}${PYTHONPATH:+:$PYTHONPATH}" \ | |
| python3 -m infx.workflows.speedbench_matrix \ | |
| --result-dir speedbench_results \ | |
| --model-key "$(basename "$(echo "$MODEL" | tr '[:upper:]' '[:lower:]')")" \ | |
| --thinking-modes "$THINKING_MODES" \ | |
| --mtp-list "$MTP_LIST" \ | |
| > speedbench-reference-al.yaml | |
| if [ ! -f "speedbench-reference-al.yaml" ]; then | |
| echo "AL collection failed: speedbench-reference-al.yaml not produced." >&2 | |
| exit 1 | |
| fi | |
| echo "### SpeedBench AL matrix" >> "$GITHUB_STEP_SUMMARY" | |
| echo '```yaml' >> "$GITHUB_STEP_SUMMARY" | |
| cat speedbench-reference-al.yaml >> "$GITHUB_STEP_SUMMARY" | |
| echo '```' >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload AL matrix artifact | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: speedbench-reference-al-${{ inputs.model-prefix }} | |
| path: speedbench-reference-al.yaml | |
| if-no-files-found: warn | |
| - name: Open PR updating reference yaml | |
| if: ${{ inputs.open-pr && success() }} | |
| env: | |
| # Repository write credential for authorized reference-data collection. | |
| GH_TOKEN: ${{ secrets.SPEEDBENCH_PAT }} # zizmor: ignore[secrets-outside-env] | |
| run: | | |
| set -eo pipefail | |
| # NOTE: the reference yaml is keyed by model at the top level. This | |
| # overwrites it with the current model's matrix; when more than one | |
| # model is collected, replace this cp with a per-model-key YAML merge. | |
| cp speedbench-reference-al.yaml benchmarks/speedbench-reference-al.yaml | |
| BRANCH="speedbench-al/${MODEL_PREFIX}-auto-${GITHUB_RUN_ID}" | |
| git config user.name "github-actions" | |
| git config user.email "github-actions@github.com" | |
| git checkout -b "$BRANCH" | |
| git add benchmarks/speedbench-reference-al.yaml | |
| if git diff --cached --quiet; then | |
| echo "No change in reference yaml; skipping PR." | |
| exit 0 | |
| fi | |
| git commit -m "Update SpeedBench AL reference matrix for ${MODEL} (auto, run ${GITHUB_RUN_ID})" | |
| git -c credential.helper= -c 'credential.helper=!gh auth git-credential' push -u origin "$BRANCH" | |
| gh pr create \ | |
| --title "Update SpeedBench AL reference matrix for ${MODEL_PREFIX} (auto)" \ | |
| --body "Auto-generated by the SpeedBench AL Collection workflow (run ${GITHUB_RUN_ID}). Model: \`${MODEL}\`, category: \`${CATEGORY}\`, MTP: \`${MTP_LIST}\`, thinking: \`${THINKING_MODES}\`, output_len: \`${SPEEDBENCH_OUTPUT_LEN}\`. Please review the measured values before merging." \ | |
| --base main \ | |
| --head "$BRANCH" | |
| - name: Upload server logs | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: speedbench_server_logs-${{ inputs.model-prefix }} | |
| path: | | |
| speedbench_results/server_*.log | |
| speedbench_results/srt-logs_*.tar.gz | |
| if-no-files-found: ignore | |
| # Per-request benchmark detail (vllm bench serve --save-detailed): includes | |
| # the model's response text per request, so the AL of each cell can be | |
| # audited for output correctness (sensible text + correct thinking mode). | |
| - name: Upload detailed benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: speedbench_detailed_results-${{ inputs.model-prefix }} | |
| path: speedbench_results/speedbench_* | |
| if-no-files-found: ignore | |
| - name: Resource cleanup (post-run) | |
| if: always() | |
| run: *resource-cleanup |