Repository navigation
SpeedBench AL Collection #163
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: SpeedBench AL Collection | |
| # Push-button (workflow_dispatch) collection of a SPEED-Bench acceptance-length | |
| # (AL) matrix: thinking_on/off x MTP levels, for the given model (defaults to | |
| # DeepSeek-V4-Pro). Produces the golden reference consumed by the | |
| # synthetic-acceptance framework and (optionally) opens a PR updating | |
| # benchmarks/speedbench-reference-al.yaml. | |
| # Independent collection requests use priority-scheduled runner slots. | |
| on: # zizmor: ignore[concurrency-limits] | |
| workflow_dispatch: | |
| inputs: | |
| runner: | |
| description: "Self-hosted GPU runner label (B300)" | |
| required: false | |
| type: string | |
| default: 'b300' | |
| model: | |
| description: "HF model id (basename must be a models.entries key of the cluster in configs/runners.yaml for pre-staged local weights)" | |
| required: false | |
| type: string | |
| default: 'deepseek-ai/DeepSeek-V4-Pro' | |
| model-prefix: | |
| description: "Model prefix; drives the exp name, collector script, and artifact names" | |
| required: false | |
| type: string | |
| default: 'dsv4' | |
| image: | |
| description: "vLLM container image" | |
| required: false | |
| type: string | |
| default: 'vllm/vllm-openai:v0.21.0' | |
| mtp-list: | |
| description: "Space-separated MTP levels (num_speculative_tokens)" | |
| required: false | |
| type: string | |
| default: '1 2 3 4 5 6 7 8' | |
| thinking-modes: | |
| description: "Space-separated thinking modes to collect" | |
| required: false | |
| type: string | |
| default: 'off on' | |
| category: | |
| description: "SPEED-Bench category" | |
| required: false | |
| type: string | |
| default: 'coding' | |
| output-len: | |
| description: "Per-request output length" | |
| required: false | |
| type: string | |
| default: '4096' | |
| thinking-kwargs: | |
| description: "chat_template_kwargs JSON for thinking-on cells (match golden config)" | |
| required: false | |
| type: string | |
| default: '{"thinking": true, "reasoning_effort": "high"}' | |
| salloc-time: | |
| description: "Slurm allocation minutes (16 server starts ~ several hours)" | |
| required: false | |
| type: string | |
| default: '480' | |
| open-pr: | |
| description: "Open a PR updating benchmarks/speedbench-reference-al.yaml (default off: artifact-only, paste values in manually)" | |
| required: false | |
| type: boolean | |
| default: false | |
| ref: | |
| description: "Git ref (branch/sha) to checkout" | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| env: | |
| PYTHONPATH: ${{ github.workspace }}/inferencex-e2e | |
| INFERENCEX_E2E_ROOT: ${{ github.workspace }} | |
| DRAFT_SAMPLE_METHOD: greedy | |
| PORT: '8888' | |
| INFMAX_CONTAINER_WORKSPACE: '/workspace' | |
| DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1' | |
| MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600' | |
| ENROOT_IMPORT_TIME_LIMIT: '120' | |
| GPU_MONITOR_INTERVAL: '1' | |
| GPU_METRICS_CSV: 'gpu_metrics.csv' | |
| KEEP_LOGS: '0' | |
| IS_MULTINODE: 'false' | |
| EVAL_ONLY: 'false' | |
| RUN_EVAL: 'false' | |
| IS_AGENTIC: '0' | |
| REQUIRE_POWER: '0' | |
| SCENARIO_SUBDIR: fixed_seq_len/ | |
| HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} | |
| HF_HUB_CACHE: '/mnt/hf_hub_cache/' | |
| # The script driver (infx/launch/drivers/script.py) runs the collector. A MODEL whose | |
| # basename is a models.entries key of the cluster serves from those pre-staged weights; | |
| # others resolve under models.download-root, where the collector downloads them. | |
| MODEL: ${{ inputs.model }} | |
| MODEL_PREFIX: ${{ inputs.model-prefix }} | |
| PRECISION: fp4 | |
| FRAMEWORK: vllm | |
| EXP_NAME: ${{ inputs.model-prefix }}_speedbench | |
| IMAGE: ${{ inputs.image }} | |
| TP: '8' | |
| EP_SIZE: '1' | |
| DP_ATTENTION: 'false' | |
| SPEC_DECODING: mtp | |
| BENCH_SCRIPT_OVERRIDE: benchmarks/single_node/speedbench/${{ inputs.model-prefix }}_fp4_b300_vllm.sh | |
| SALLOC_TIME_LIMIT: ${{ inputs.salloc-time }} | |
| # Matrix-collector tunables (propagated into the container via srun --export=ALL). | |
| MTP_LIST: ${{ inputs.mtp-list }} | |
| THINKING_MODES: ${{ inputs.thinking-modes }} | |
| CATEGORY: ${{ inputs.category }} | |
| SPEEDBENCH_OUTPUT_LEN: ${{ inputs.output-len }} | |
| CHAT_TEMPLATE_KWARGS_ON: ${{ inputs.thinking-kwargs }} | |
| OUT_YAML: /workspace/speedbench-reference-al.yaml | |
| PYTHONDONTWRITEBYTECODE: '1' | |
| PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache | |
| jobs: | |
| setup: | |
| name: setup | |
| runs-on: ubuntu-latest | |
| outputs: | |
| priority: ${{ steps.score.outputs.priority }} | |
| queue-token: ${{ steps.score.outputs.queue-token }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: { clean: true, persist-credentials: false } | |
| - name: Checkout the measured revision's launcher | |
| if: ${{ inputs.ref && inputs.ref != '' }} | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ inputs.ref }} | |
| path: .measured | |
| sparse-checkout: inferencex-e2e/infx/launch | |
| persist-credentials: false | |
| - name: Require the Python launcher | |
| if: ${{ inputs.ref && inputs.ref != '' }} | |
| # The collect job runs the measured revision's own `python -m infx.launch`. | |
| run: python3 -P -m infx.workflows.require_launcher .measured | |
| - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| - id: score | |
| env: | |
| RUNNER: ${{ inputs.runner }} | |
| MODEL_PREFIX: ${{ inputs.model-prefix }} | |
| run: | | |
| entry=$(jq -cn \ | |
| --arg runner "$RUNNER" \ | |
| --arg model_prefix "$MODEL_PREFIX" \ | |
| '[{ | |
| runner: $runner, | |
| framework: "vllm", | |
| "model-prefix": $model_prefix, | |
| precision: "fp4", | |
| "spec-decoding": "mtp" | |
| }]') | |
| scored=$(printf '%s' "$entry" | | |
| uv run --no-project --exclude-newer PT12H --python 3.12 --with pyyaml \ | |
| python -m infx.workflows.ci_priority \ | |
| --queue-namespace "${GITHUB_RUN_ID}:${GITHUB_RUN_ATTEMPT}") | |
| echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT" | |
| echo "queue-token=$(jq -r '.[0]["queue-token"]' <<<"$scored")" >> "$GITHUB_OUTPUT" | |
| collect-al: | |
| needs: setup | |
| runs-on: >- | |
| ${{ fromJSON( | |
| vars.PRIORITY_SCHEDULER_ENABLED == 'true' && | |
| ( | |
| vars.NODE_SLOT_SCHEDULER_ENABLED == 'true' && | |
| format( | |
| '["self-hosted",{0},{1},{2},{3}]', | |
| toJSON(inputs.runner), | |
| '"nodes:1"', | |
| toJSON(format( | |
| 'ci-job-{0}-{1}', | |
| needs.setup.outputs.priority, | |
| needs.setup.outputs.queue-token | |
| )), | |
| toJSON(format('ci-attempt-{0}', github.run_attempt)) | |
| ) || | |
| format( | |
| '["self-hosted",{0},{1},{2}]', | |
| toJSON(inputs.runner), | |
| toJSON(format( | |
| 'ci-job-{0}-{1}', | |
| needs.setup.outputs.priority, | |
| needs.setup.outputs.queue-token | |
| )), | |
| toJSON(format('ci-attempt-{0}', github.run_attempt)) | |
| ) | |
| ) || | |
| format('[{0}]', toJSON(inputs.runner)) | |
| ) }} | |
| timeout-minutes: 600 | |
| name: "p${{ needs.setup.outputs.priority }} | SpeedBench AL matrix | ${{ inputs.category }} | mtp=[${{ inputs.mtp-list }}] | thinking=[${{ inputs.thinking-modes }}]" | |
| steps: | |
| - name: Resource cleanup (pre-run) | |
| run: &resource-cleanup | | |
| # Cleanup SLURM resources | |
| if command -v squeue >/dev/null 2>&1; then | |
| # srtctl submits as inferencex-<runner> (infx.launch.backends.slurm.srtctl_job_name). | |
| for job_name in "${RUNNER_NAME}" "inferencex-${RUNNER_NAME}"; do | |
| echo "[Slurm] Cleaning up jobs with name: ${job_name} ..." | |
| scancel --name="${job_name}" || true | |
| while [ -n "$(squeue --name="${job_name}" --noheader --format='%i')" ]; do | |
| squeue --name="${job_name}" | |
| sleep 5 | |
| done | |
| done | |
| fi | |
| # Cleanup AL-matrix outputs from a prior job on this runner so a stale | |
| # matrix from a previous run is never picked up as this job's output. | |
| rm -rf "${GITHUB_WORKSPACE}/speedbench_results" \ | |
| "${GITHUB_WORKSPACE}/inferencex-e2e/speedbench_results" 2>/dev/null || true | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ inputs.ref || github.sha }} | |
| clean: true | |
| submodules: true | |
| persist-credentials: false | |
| - name: Cleanup stale outputs (pre-run) | |
| run: | | |
| if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi | |
| rm -f speedbench-reference-al.yaml || true | |
| rm -f gpu_metrics.csv || true | |
| rm -rf speed_bench_data || true | |
| - name: Checkout launch tooling | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.workflow_sha }} | |
| path: .launch-tooling | |
| persist-credentials: false | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| with: | |
| enable-cache: false | |
| - name: Prepare launcher Python | |
| # Keep uv's cache and Pythons per runner: some runner homes can't host uv's defaults. | |
| env: | |
| UV_CACHE_DIR: ${{ github.workspace }}/../.infx-uv-cache | |
| UV_PYTHON_INSTALL_DIR: ${{ github.workspace }}/../.infx-uv-python | |
| run: | | |
| # Never activated, so launched Slurm jobs inherit the runner's PATH and no VIRTUAL_ENV. | |
| launch_venv="$RUNNER_TEMP/infx-launch-venv" | |
| uv venv --quiet --python 3.12 "$launch_venv" | |
| uv pip install --quiet --exclude-newer PT12H --python "$launch_venv/bin/python" \ | |
| -r "$GITHUB_WORKSPACE/.launch-tooling/inferencex-e2e/pyproject.toml" | |
| echo "INFERENCEX_LAUNCH_PYTHON=$launch_venv/bin/python" >> "$GITHUB_ENV" | |
| - name: Launcher cleanup (pre-run) | |
| env: | |
| PYTHONPATH: ${{ github.workspace }}/.launch-tooling/inferencex-e2e | |
| # Second pass after the shell cancel, which needs no Python: the cluster's backend | |
| # removes what this runner left behind. | |
| run: &launcher-cleanup | | |
| "$INFERENCEX_LAUNCH_PYTHON" -P -m infx.launch cleanup | |
| - name: Collect AL matrix | |
| env: | |
| RUNNER_NAME: ${{ runner.name }} | |
| run: | | |
| if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi | |
| export GITHUB_WORKSPACE="$PWD" | |
| echo "INFERENCEX_E2E_ROOT=$PWD" >> "$GITHUB_ENV" | |
| set -eo pipefail | |
| export GPU_COUNT="$TP" | |
| source benchmarks/runtime_settings.sh | |
| "$INFERENCEX_LAUNCH_PYTHON" -m infx.launch run | |
| if [ ! -f "speedbench-reference-al.yaml" ]; then | |
| echo "AL collection failed: speedbench-reference-al.yaml not produced." >&2 | |
| exit 1 | |
| fi | |
| echo "### SpeedBench AL matrix" >> "$GITHUB_STEP_SUMMARY" | |
| echo '```yaml' >> "$GITHUB_STEP_SUMMARY" | |
| cat speedbench-reference-al.yaml >> "$GITHUB_STEP_SUMMARY" | |
| echo '```' >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload AL matrix artifact | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: speedbench-reference-al-${{ inputs.model-prefix }} | |
| path: ${{ env.INFERENCEX_E2E_ROOT }}/speedbench-reference-al.yaml | |
| if-no-files-found: warn | |
| - name: Open PR updating reference yaml | |
| if: ${{ inputs.open-pr && success() }} | |
| env: | |
| # Repository write credential for authorized reference-data collection. | |
| GH_TOKEN: ${{ secrets.SPEEDBENCH_PAT }} # zizmor: ignore[secrets-outside-env] | |
| run: | | |
| if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi | |
| set -eo pipefail | |
| # NOTE: the reference yaml is keyed by model at the top level. This | |
| # overwrites it with the current model's matrix; when more than one | |
| # model is collected, replace this cp with a per-model-key YAML merge. | |
| cp speedbench-reference-al.yaml benchmarks/speedbench-reference-al.yaml | |
| BRANCH="speedbench-al/${MODEL_PREFIX}-auto-${GITHUB_RUN_ID}" | |
| git config user.name "github-actions" | |
| git config user.email "github-actions@github.com" | |
| git checkout -b "$BRANCH" | |
| git add benchmarks/speedbench-reference-al.yaml | |
| if git diff --cached --quiet; then | |
| echo "No change in reference yaml; skipping PR." | |
| exit 0 | |
| fi | |
| git commit -m "Update SpeedBench AL reference matrix for ${MODEL} (auto, run ${GITHUB_RUN_ID})" | |
| git -c credential.helper= -c 'credential.helper=!gh auth git-credential' push -u origin "$BRANCH" | |
| gh pr create \ | |
| --title "Update SpeedBench AL reference matrix for ${MODEL_PREFIX} (auto)" \ | |
| --body "Auto-generated by the SpeedBench AL Collection workflow (run ${GITHUB_RUN_ID}). Model: \`${MODEL}\`, category: \`${CATEGORY}\`, MTP: \`${MTP_LIST}\`, thinking: \`${THINKING_MODES}\`, output_len: \`${SPEEDBENCH_OUTPUT_LEN}\`. Please review the measured values before merging." \ | |
| --base main \ | |
| --head "$BRANCH" | |
| - name: Upload server logs | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: speedbench_server_logs-${{ inputs.model-prefix }} | |
| path: ${{ env.INFERENCEX_E2E_ROOT }}/speedbench_results/server_*.log | |
| if-no-files-found: ignore | |
| # Per-request benchmark detail (vllm bench serve --save-detailed): includes | |
| # the model's response text per request, so the AL of each cell can be | |
| # audited for output correctness (sensible text + correct thinking mode). | |
| - name: Upload detailed benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: speedbench_detailed_results-${{ inputs.model-prefix }} | |
| path: ${{ env.INFERENCEX_E2E_ROOT }}/speedbench_results/speedbench_* | |
| if-no-files-found: ignore | |
| - name: Resource cleanup (post-run) | |
| if: always() | |
| run: *resource-cleanup | |
| - name: Launcher cleanup (post-run) | |
| if: ${{ always() && env.INFERENCEX_LAUNCH_PYTHON != '' }} | |
| env: | |
| PYTHONPATH: ${{ github.workspace }}/.launch-tooling/inferencex-e2e | |
| run: *launcher-cleanup |