Repository navigation
fix(agentic): require GB200 validated checkpoints #3
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Template - Multi-Node Benchmark | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| runner: | ||
| required: true | ||
| type: string | ||
| image: | ||
| required: true | ||
| type: string | ||
| model: | ||
| required: true | ||
| type: string | ||
| model-prefix: | ||
| required: true | ||
| type: string | ||
| framework: | ||
| required: true | ||
| type: string | ||
| precision: | ||
| required: true | ||
| type: string | ||
| exp-name: | ||
| required: true | ||
| type: string | ||
| isl: | ||
| required: true | ||
| type: string | ||
| osl: | ||
| required: true | ||
| type: string | ||
| conc-list: | ||
| required: true | ||
| type: string | ||
| spec-decoding: | ||
| required: true | ||
| type: string | ||
| disagg: | ||
| required: true | ||
| type: string | ||
| max-model-len: | ||
| required: true | ||
| type: string | ||
| prefill-num-worker: | ||
| required: true | ||
| type: string | ||
| prefill-tp: | ||
| required: true | ||
| type: string | ||
| prefill-ep: | ||
| required: true | ||
| type: string | ||
| prefill-dp-attn: | ||
| required: true | ||
| type: string | ||
| prefill-additional-settings: | ||
| required: false | ||
| type: string | ||
| default: "[]" | ||
| decode-num-worker: | ||
| required: true | ||
| type: string | ||
| decode-tp: | ||
| required: true | ||
| type: string | ||
| decode-ep: | ||
| required: true | ||
| type: string | ||
| decode-dp-attn: | ||
| required: true | ||
| type: string | ||
| decode-additional-settings: | ||
| required: false | ||
| type: string | ||
| default: "[]" | ||
| run-eval: | ||
| type: boolean | ||
| required: false | ||
| default: false | ||
| eval-only: | ||
| description: "Run only evals (skip throughput benchmark)" | ||
| type: boolean | ||
| required: false | ||
| default: false | ||
| eval-conc: | ||
| description: "Concurrency to use for eval requests (overrides default max-of-conc-list)" | ||
| type: string | ||
| required: false | ||
| default: "" | ||
| scenario-type: | ||
| description: "Scenario type (fixed-seq-len or agentic-coding)" | ||
| type: string | ||
| required: false | ||
| default: fixed-seq-len | ||
| conc: | ||
| description: "First concurrency for agentic-coding scenarios; CONC_LIST carries the full batch" | ||
| type: string | ||
| required: false | ||
| default: "" | ||
| duration: | ||
| description: "Agentic trace replay duration in seconds" | ||
| type: string | ||
| required: false | ||
| default: "1800" | ||
| offloading: | ||
| description: "KV offload backend for agentic scenarios (none/cpu/ssd)" | ||
| required: false | ||
| type: string | ||
| default: 'none' | ||
| total-cpu-dram-gb: | ||
| description: "Total CPU DRAM in GB for KV offloading" | ||
| required: false | ||
| type: string | ||
| default: '600' | ||
| ref: | ||
| description: "Git ref (branch/sha) to checkout" | ||
| required: false | ||
| type: string | ||
| env: | ||
| RANDOM_RANGE_RATIO: 0.8 | ||
| EXP_NAME: ${{ inputs.exp-name }} | ||
| IMAGE: ${{ inputs.image }} | ||
| MODEL_PREFIX: ${{ inputs.model-prefix }} | ||
| MODEL: ${{ inputs.model }} | ||
| FRAMEWORK: ${{ inputs.framework }} | ||
| PRECISION: ${{ inputs.precision }} | ||
| ISL: ${{ inputs.isl }} | ||
| OSL: ${{ inputs.osl }} | ||
| MAX_MODEL_LEN: ${{ inputs.max-model-len }} | ||
| CONC_LIST: ${{ join(fromJson(inputs.conc-list), ' ') }} | ||
| SPEC_DECODING: ${{ inputs.spec-decoding }} | ||
| DISAGG: ${{ inputs.disagg }} | ||
| RUN_EVAL: ${{ inputs.run-eval }} | ||
| EVAL_ONLY: ${{ inputs.eval-only }} | ||
| EVAL_CONC: ${{ inputs.eval-conc }} | ||
| SCENARIO_TYPE: ${{ inputs.scenario-type }} | ||
| SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }} | ||
| IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }} | ||
| CONC: ${{ inputs.conc }} | ||
| DURATION: ${{ inputs.duration }} | ||
| OFFLOADING: ${{ inputs.offloading }} | ||
| TOTAL_CPU_DRAM_GB: ${{ inputs.total-cpu-dram-gb }} | ||
| PYTHONDONTWRITEBYTECODE: '1' | ||
| PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache | ||
| PREFILL_NUM_WORKERS: ${{ inputs.prefill-num-worker }} | ||
| PREFILL_TP: ${{ inputs.prefill-tp }} | ||
| PREFILL_EP: ${{ inputs.prefill-ep }} | ||
| PREFILL_DP_ATTN: ${{ inputs.prefill-dp-attn }} | ||
| DECODE_NUM_WORKERS: ${{ inputs.decode-num-worker }} | ||
| DECODE_TP: ${{ inputs.decode-tp }} | ||
| DECODE_EP: ${{ inputs.decode-ep }} | ||
| DECODE_DP_ATTN: ${{ inputs.decode-dp-attn }} | ||
| permissions: | ||
| contents: read | ||
| jobs: | ||
| benchmark: | ||
| runs-on: ${{ inputs.runner }} | ||
| timeout-minutes: 480 | ||
| name: "${{ inputs.exp-name }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework }} | P(tp${{ inputs.prefill-tp }}/ep${{ inputs.prefill-ep }}/dp${{ inputs.prefill-dp-attn }}/nw${{ inputs.prefill-num-worker }}) D(tp${{ inputs.decode-tp }}/ep${{ inputs.decode-ep }}/dp${{ inputs.decode-dp-attn }}/nw${{ inputs.decode-num-worker }}) | disagg-${{ inputs.disagg }} spec-${{ inputs.spec-decoding }} conc-${{ join(fromJson(inputs.conc-list), 'x') }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }}" | ||
| steps: | ||
| - name: Slurm cleanup (pre-run) | ||
| run: &slurm-cleanup | | ||
| if command -v squeue >/dev/null 2>&1; then | ||
| # Clean both the bare runner name and the "ifx-" prefixed form. | ||
| # launch_gb200-nv.sh names jobs ifx-<runner> to dodge a foreign | ||
| # runner fleet on watchtower that scancels by the bare name | ||
| # across users (see the comment there). squeue is filtered to | ||
| # our user so the wait loop can't hang on a same-named foreign | ||
| # job we have no permission to cancel. | ||
| echo "[Slurm] Cleaning up jobs named: ${{ runner.name }}, ifx-${{ runner.name }} ..." | ||
| scancel --name="${{ runner.name }}" || true | ||
| scancel --name="ifx-${{ runner.name }}" || true | ||
| while [ -n "$(squeue --user="$USER" --name='${{ runner.name }},ifx-${{ runner.name }}' --noheader --format='%i')" ]; do | ||
| squeue --user="$USER" --name="${{ runner.name }},ifx-${{ runner.name }}" | ||
| sleep 5 | ||
| done | ||
| fi | ||
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | ||
| with: | ||
| token: ${{ secrets.REPO_PAT }} | ||
| fetch-depth: 0 | ||
| ref: ${{ inputs.ref || github.sha }} | ||
| clean: true | ||
| submodules: true | ||
| - name: Set result identity | ||
| id: result-identity | ||
| env: | ||
| # AgentX checkpoints must survive rerouting to another physical | ||
| # runner on `rerun --failed`, so use the stable runner label there. | ||
| # Other scenarios retain the historical physical-runner suffix. | ||
| RESULT_FILENAME: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_prefill-tp${{ env.PREFILL_TP }}-ep${{ env.PREFILL_EP }}-dp${{ env.PREFILL_DP_ATTN }}-nw${{ env.PREFILL_NUM_WORKERS }}_decode-tp${{ env.DECODE_TP }}-ep${{ env.DECODE_EP }}-dp${{ env.DECODE_DP_ATTN }}-nw${{ env.DECODE_NUM_WORKERS }}_disagg-${{ env.DISAGG }}_spec-${{ env.SPEC_DECODING }}_conc${{ join(fromJson(inputs.conc-list), 'x') }}_${{ inputs.scenario-type == 'agentic-coding' && inputs.runner || runner.name }} | ||
| run: | | ||
| echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV" | ||
| echo "artifact-key=${RESULT_FILENAME}" >> "$GITHUB_OUTPUT" | ||
| - name: Restore prior AgentX concurrency checkpoints | ||
| if: ${{ inputs.scenario-type == 'agentic-coding' && github.run_attempt > 1 }} | ||
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | ||
| with: | ||
| pattern: checkpoint_agentic_${{ steps.result-identity.outputs.artifact-key }}_attempt-* | ||
| path: agentic_checkpoints | ||
| merge-multiple: true | ||
| - name: Cleanup stale eval outputs (pre-run) | ||
| if: ${{ inputs.run-eval || inputs.eval-only }} | ||
| run: | | ||
| rm -f meta_env.json || true | ||
| rm -f results*.json || true | ||
| rm -f sample*.jsonl || true | ||
| - name: Launch multi-node job script | ||
| env: | ||
| RUNNER_NAME: ${{ runner.name }} | ||
| RUNNER_TYPE: ${{ inputs.runner }} | ||
| run: | | ||
| set -x | ||
| export ${{ join(fromJson(inputs.prefill-additional-settings), ' ') }} ${{ join(fromJson(inputs.decode-additional-settings), ' ') }} | ||
| export IS_MULTINODE=true | ||
| bash ./runners/launch_${RUNNER_NAME%%_*}.sh | ||
| if [ "${{ inputs.eval-only }}" = "true" ]; then | ||
| echo "Eval-only mode: skipping benchmark result file check" | ||
| # Verify eval produced results | ||
| if ! ls results*.json 1>/dev/null 2>&1; then | ||
| echo "Eval-only run failed: no results*.json files found." >&2 | ||
| exit 1 | ||
| fi | ||
| elif [ "${{ inputs.scenario-type }}" = "agentic-coding" ]; then | ||
| expected_count=$(wc -w <<< "$CONC_LIST" | tr -d ' ') | ||
| shopt -s nullglob | ||
| agentic_results=("${RESULT_FILENAME}"_conc*.json) | ||
| shopt -u nullglob | ||
| if [ "${#agentic_results[@]}" -ne "$expected_count" ]; then | ||
| echo "Run failed: expected $expected_count agentic results, found ${#agentic_results[@]}." >&2 | ||
| exit 1 | ||
| fi | ||
| # Existence is not enough: process_agentic_result.py writes the | ||
| # aggregate even when aiperf recorded zero valid requests. Require | ||
| # successful requests and the success marker staged only after | ||
| # replay/result validation in every concurrency result. | ||
| for result_file in "${agentic_results[@]}"; do | ||
| echo "Found agentic result file: $result_file" | ||
| conc=$(sed -n 's/.*_conc\([0-9][0-9]*\)\.json$/\1/p' <<< "$result_file") | ||
| marker_file="agentic_checkpoints/${RESULT_FILENAME}_conc${conc}.success.json" | ||
| checkpoint_file="agentic_checkpoints/${RESULT_FILENAME}_conc${conc}.json" | ||
| if [ -z "$conc" ] || [ ! -f "$marker_file" ] || [ ! -f "$checkpoint_file" ]; then | ||
| echo "Run failed: $result_file is missing validated AgentX checkpoint marker $marker_file." >&2 | ||
| exit 1 | ||
| fi | ||
| ok=$(python3 - "$result_file" "$checkpoint_file" "$marker_file" "$RESULT_FILENAME" "$conc" <<'PY' 2>/dev/null || echo 0 | ||
| import json | ||
| import sys | ||
| result_file, checkpoint_file, marker_file, base_result_filename, concurrency_raw = sys.argv[1:] | ||
| concurrency = int(concurrency_raw) | ||
| with open(result_file) as handle: | ||
| result = json.load(handle) | ||
| with open(checkpoint_file) as handle: | ||
| checkpoint = json.load(handle) | ||
| with open(marker_file) as handle: | ||
| marker = json.load(handle) | ||
| expected_marker = { | ||
| "schema_version": 1, | ||
| "base_result_filename": base_result_filename, | ||
| "result_filename": f"{base_result_filename}_conc{concurrency}.json", | ||
| "concurrency": concurrency, | ||
| } | ||
| print( | ||
| int( | ||
| marker == expected_marker | ||
| and int(result.get("conc", -1)) == concurrency | ||
| and int(checkpoint.get("conc", -1)) == concurrency | ||
| and int(result.get("num_requests_successful", 0)) > 0 | ||
| and int(checkpoint.get("num_requests_successful", 0)) > 0 | ||
| ) | ||
| ) | ||
| PY | ||
| ) | ||
| if [ "$ok" != "1" ]; then | ||
| echo "Run failed: $result_file does not match a validated AgentX checkpoint." >&2 | ||
| exit 1 | ||
| fi | ||
| done | ||
| else | ||
| # Check if at least one result file was created | ||
| if ls ${RESULT_FILENAME}_*.json 1> /dev/null 2>&1; then | ||
| echo "RESULT_FILENAME=${RESULT_FILENAME}" >> $GITHUB_ENV | ||
| echo "Found result files: $(ls ${RESULT_FILENAME}_*.json)" | ||
| else | ||
| echo "Run failed: No benchmark result files found for ${RESULT_FILENAME}_*.json" >&2 | ||
| exit 1 | ||
| fi | ||
| fi | ||
| - name: Process result | ||
| if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} | ||
| env: | ||
| RUNNER_TYPE: ${{ inputs.runner }} | ||
| run: | | ||
| # Process each result file | ||
| for result_file in ${RESULT_FILENAME}_*.json; do | ||
| if [ -f "$result_file" ]; then | ||
| echo "Processing $result_file" | ||
| # Extract GPU count, prefill_gpus and decode_gpus from filename for tp_size calculation | ||
| gpus=$(echo "$result_file" | sed -n "s/.*_gpus_\([0-9][0-9]*\).*\.json/\1/p") | ||
| prefill_gpus=$(echo "$result_file" | sed -n "s/.*_ctx_\([0-9]*\).*\.json/\1/p") | ||
| decode_gpus=$(echo "$result_file" | sed -n "s/.*_gen_\([0-9]*\).*\.json/\1/p") | ||
| if [ -n "$gpus" ]; then | ||
| # Non-disagg results lack _ctx_/_gen_ fields; all GPUs belong to the single worker | ||
| if [ -z "$prefill_gpus" ] || [ -z "$decode_gpus" ]; then | ||
| prefill_gpus=$gpus | ||
| decode_gpus=0 | ||
| fi | ||
| echo "Extracted: gpus=$gpus, prefill_gpus=$prefill_gpus, decode_gpus=$decode_gpus" | ||
| RESULT_FILENAME=${result_file%.json} IS_MULTINODE=true PREFILL_GPUS="$prefill_gpus" DECODE_GPUS="$decode_gpus" python3 utils/process_result.py | ||
| fi | ||
| fi | ||
| done | ||
| - name: Upload result | ||
| if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }} | ||
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | ||
| with: | ||
| name: bmk_${{ env.RESULT_FILENAME }} | ||
| path: agg_${{ env.RESULT_FILENAME }}_*.json | ||
| - name: Upload server logs | ||
| if: always() | ||
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | ||
| with: | ||
| name: multinode_server_logs_${{ env.RESULT_FILENAME }} | ||
| path: multinode_server_logs.tar.gz | ||
| if-no-files-found: ignore | ||
| - name: Upload AgentX concurrency checkpoints | ||
| if: ${{ always() && inputs.scenario-type == 'agentic-coding' }} | ||
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | ||
| with: | ||
| name: checkpoint_agentic_${{ steps.result-identity.outputs.artifact-key }}_attempt-${{ github.run_attempt }} | ||
| path: agentic_checkpoints/${{ env.RESULT_FILENAME }}_conc* | ||
| if-no-files-found: ignore | ||
| - name: Upload agentic aggregated result | ||
| if: ${{ !inputs.eval-only && inputs.scenario-type == 'agentic-coding' }} | ||
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | ||
| with: | ||
| name: bmk_agentic_${{ env.RESULT_FILENAME }} | ||
| path: ${{ env.RESULT_FILENAME }}_conc*.json | ||
| - name: Upload agentic raw results | ||
| if: ${{ always() && inputs.scenario-type == 'agentic-coding' }} | ||
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | ||
| with: | ||
| name: agentic_${{ env.RESULT_FILENAME }} | ||
| path: | | ||
| LOGS/agentic/conc_*/benchmark.log | ||
| LOGS/agentic/conc_*/benchmark_command.txt | ||
| LOGS/agentic/conc_*/workload_distribution_summary.txt | ||
| LOGS/agentic/conc_*/workload_distribution_plots.png | ||
| LOGS/agentic/conc_*/aiperf_artifacts/profile_export_aiperf.json | ||
| LOGS/agentic/conc_*/aiperf_artifacts/detailed_results.csv | ||
| LOGS/agentic/conc_*/aiperf_artifacts/debug_trace.jsonl | ||
| if-no-files-found: ignore | ||
| - name: Upload eval results (if any) | ||
| if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }} | ||
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | ||
| with: | ||
| name: eval_${{ env.EXP_NAME }}_${{ env.RESULT_FILENAME }} | ||
| path: | | ||
| meta_env.json | ||
| results*.json | ||
| sample*.jsonl | ||
| if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }} | ||
| - name: Verify eval scores | ||
| if: ${{ (success() || failure()) && inputs.eval-only }} | ||
| run: python3 utils/evals/validate_scores.py | ||
| - name: Cleanup eval outputs (post-upload) | ||
| if: ${{ always() && (inputs.run-eval || inputs.eval-only) }} | ||
| run: | | ||
| rm -f meta_env.json || true | ||
| rm -f results*.json || true | ||
| rm -f sample*.jsonl || true | ||
| - name: Slurm cleanup (post-run) | ||
| if: always() | ||
| run: *slurm-cleanup | ||
