benchmark-multinode-tmpl.yml

name: Template - Multi-Node Benchmark

on:
  workflow_call:
    secrets:
      INFERENCEX_OFFICIAL_RO_HF_TOKEN:
        required: true
    inputs:
      config:
        description: "Benchmark configuration as JSON"
        required: true
        type: string
      klaud-run:
        description: "Run Klaud Cold benchmarks at background priority"
        required: false
        type: boolean
        default: false
      runner:
        required: true
        type: string
      node-count:
        description: "Total Slurm nodes required by this benchmark"
        required: true
        type: number
      priority:
        description: "Higher-is-sooner CI priority score"
        required: true
        type: string
      queue-token:
        description: "One-shot queue token generated for this job"
        required: true
        type: string
      conc-list:
        description: "JSON concurrency list; AgentX throughput requires exactly one value per server deployment"
        required: true
        type: string
      run-eval:
        type: boolean
        required: false
        default: false
      eval-only:
        description: "Run only evals (skip throughput benchmark)"
        type: boolean
        required: false
        default: false
      eval-framework:
        description: "Eval runner (lm-eval, kimi-vendor, minimax-vendor, or bfcl)"
        type: string
        required: false
        default: "lm-eval"
      eval-suite:
        description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval"
        type: string
        required: false
        default: ""
      eval-conc:
        description: "Concurrency value or space-separated list for eval requests (overrides default max-of-conc-list)"
        type: string
        required: false
        default: ""
      eval-limit:
        description: "Eval instance count: empty = whole split (default); N = first-N smoke slice"
        required: false
        type: string
        default: ""
      require-power:
        description: "Fail fixed-sequence result processing when GPU power is invalid"
        type: boolean
        required: false
        default: false
      power-producer-sha:
        description: "Expected srt-slurm producer git commit for power validation"
        type: string
        required: false
        default: ''
      # Agentic-coding inputs. Fixed-seq-len jobs leave these empty.
      scenario-type:
        description: "Scenario type (fixed-seq-len or agentic-coding)"
        type: string
        required: false
        default: fixed-seq-len
      conc:
        description: "AgentX throughput concurrency; must equal the sole value in conc-list"
        type: string
        required: false
        default: ""
      duration:
        description: "Agentic trace replay duration in seconds"
        type: string
        required: false
        default: "3600"
      agentx-fast:
        description: "Use one warmup request per AgentX lane and a 20-minute profile"
        type: boolean
        required: false
        default: false
      total-cpu-dram-gb:
        description: "Total CPU DRAM in GB for KV offloading"
        required: false
        type: string
        default: '600'
      ref:
        description: "Git ref (branch/sha) to checkout"
        required: false
        type: string

env:
  PYTHONPATH: ${{ github.workspace }}/inferencex-e2e
  INFERENCEX_E2E_ROOT: ${{ github.workspace }}
  PORT: '8888'
  INFMAX_CONTAINER_WORKSPACE: '/workspace'
  DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1'
  MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600'
  ENROOT_IMPORT_TIME_LIMIT: '120'
  SALLOC_TIME_LIMIT: '480'
  KEEP_LOGS: '0'
  IS_MULTINODE: 'true'
  RANDOM_RANGE_RATIO: 0.8
  # Day-zero models resolved via hf: ids download from the Hub inside the
  # slurm job (srtctl pre-download + dynamo hub fetch). Anonymous requests
  # get 429-rate-limited when several workers pull a 444 GB snapshot at
  # once; sbatch/srun inherit this env so the token reaches the workers.
  HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
  EXP_NAME: ${{ fromJSON(inputs.config).exp-name }}
  RECIPE_FINGERPRINT: ${{ fromJSON(inputs.config)['recipe-fingerprint'] || '' }}
  SRT_RECIPE: ${{ fromJSON(inputs.config)['srt-recipe'] || '' }}
  EVAL_SRT_RECIPE: ${{ fromJSON(inputs.config)['eval-srt-recipe'] || '' }}
  IMAGE: ${{ fromJSON(inputs.config).image }}
  THINKING_MODE: thinking_on
  MODEL_PREFIX: ${{ fromJSON(inputs.config).model-prefix }}
  MODEL: ${{ fromJSON(inputs.config).model }}
  FRAMEWORK: ${{ fromJSON(inputs.config).framework }}
  PRECISION: ${{ fromJSON(inputs.config).precision }}
  ISL: ${{ inputs.scenario-type == 'agentic-coding' && '0' || fromJSON(inputs.config).isl }}
  OSL: ${{ inputs.scenario-type == 'agentic-coding' && '0' || fromJSON(inputs.config).osl }}
  MAX_MODEL_LEN: ${{ inputs.scenario-type == 'agentic-coding' && '0' || fromJSON(inputs.config).max-model-len }}
  CONC_LIST: ${{ join(fromJson(inputs.conc-list), ' ') }}
  SPEC_DECODING: ${{ fromJSON(inputs.config).spec-decoding }}
  DISAGG: ${{ fromJSON(inputs.config).disagg }}
  PREFILL_HARDWARE: ${{ fromJSON(inputs.config).prefill.hardware }}
  DECODE_HARDWARE: ${{ fromJSON(inputs.config).decode.hardware }}
  RUN_EVAL: ${{ inputs.run-eval }}
  EVAL_ONLY: ${{ inputs.eval-only }}
  EVAL_FRAMEWORK: ${{ inputs.eval-framework }}
  EVAL_SUITE: ${{ inputs.eval-suite }}
  EVAL_CONC: ${{ inputs.eval-conc }}
  EVAL_LIMIT: ${{ inputs.eval-limit }}
  POWER: ${{ fromJSON(inputs.config).power && '1' || '0' }}
  REQUIRE_POWER: ${{ inputs.require-power && '1' || '0' }}
  POWER_PRODUCER_SHA: ${{ inputs.power-producer-sha }}
  # Agentic-coding env. Fixed-seq-len jobs leave these empty.
  SCENARIO_TYPE: ${{ inputs.scenario-type }}
  SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }}
  IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }}
  CONC: ${{ inputs.conc }}
  DURATION: ${{ inputs.duration }}
  AIPERF_EXPERIMENTAL_FAST: ${{ inputs.agentx-fast && '1' || '0' }}
  KV_OFFLOADING: ${{ inputs.scenario-type == 'agentic-coding' && fromJSON(inputs.config).kv-offloading || '' }}
  KV_OFFLOAD_BACKEND: ${{ inputs.scenario-type == 'agentic-coding' && fromJSON(inputs.config)['kv-offload-backend'].name || '' }}
  KV_OFFLOAD_BACKEND_METADATA: ${{ inputs.scenario-type == 'agentic-coding' && fromJSON(inputs.config)['kv-offload-backend'] && toJSON(fromJSON(inputs.config)['kv-offload-backend']) || '' }}
  ROUTER_METADATA: ${{ fromJSON(inputs.config).router && toJSON(fromJSON(inputs.config).router) || '' }}
  KV_P2P_TRANSFER: ${{ fromJSON(inputs.config)['kv-p2p-transfer'] || '' }}
  TOTAL_CPU_DRAM_GB: ${{ inputs.total-cpu-dram-gb }}
  PYTHONDONTWRITEBYTECODE: '1'
  PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache

  PREFILL_NUM_WORKERS: ${{ fromJSON(inputs.config).prefill.num-worker }}
  PREFILL_TP: ${{ fromJSON(inputs.config).prefill.tp }}
  PREFILL_PP_SIZE: ${{ fromJSON(inputs.config).prefill.pp }}
  PREFILL_DCP_SIZE: ${{ fromJSON(inputs.config).prefill.dcp-size }}
  PREFILL_PCP_SIZE: ${{ fromJSON(inputs.config).prefill.pcp-size }}
  PREFILL_DWDP_SIZE: ${{ fromJSON(inputs.config).prefill.dwdp-size || 1 }}
  DWDP_ARTIFACT_SUFFIX: ${{ fromJSON(inputs.config).prefill.dwdp-size > 1 && format('-dwdp{0}', fromJSON(inputs.config).prefill.dwdp-size) || '' }}
  PREFILL_EP: ${{ fromJSON(inputs.config).prefill.ep }}
  PREFILL_DP_ATTN: ${{ fromJSON(inputs.config).prefill.dp-attn }}

  DECODE_NUM_WORKERS: ${{ fromJSON(inputs.config).decode.num-worker }}
  DECODE_TP: ${{ fromJSON(inputs.config).decode.tp }}
  DECODE_PP_SIZE: ${{ fromJSON(inputs.config).decode.pp }}
  DECODE_DCP_SIZE: ${{ fromJSON(inputs.config).decode.dcp-size }}
  DECODE_PCP_SIZE: ${{ fromJSON(inputs.config).decode.pcp-size }}
  DECODE_EP: ${{ fromJSON(inputs.config).decode.ep }}
  DECODE_DP_ATTN: ${{ fromJSON(inputs.config).decode.dp-attn }}

permissions:
  contents: read

jobs:
  benchmark:
    runs-on: >-
      ${{ fromJSON(
        vars.PRIORITY_SCHEDULER_ENABLED == 'true' &&
        (
          vars.NODE_SLOT_SCHEDULER_ENABLED == 'true' &&
          format(
            '["self-hosted",{0},{1},{2},{3}]',
            toJSON(inputs.runner),
            toJSON(format('nodes:{0}', inputs.node-count)),
            toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)),
            toJSON(format('ci-attempt-{0}', github.run_attempt))
          ) ||
          format(
            '["self-hosted",{0},{1},{2}]',
            toJSON(inputs.runner),
            toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)),
            toJSON(format('ci-attempt-{0}', github.run_attempt))
          )
        ) ||
        format('[{0}]', toJSON(inputs.runner))
      ) }}
    # Full-context AgentX warmup can legitimately exceed the fixed-sequence
    # eight-hour envelope at high concurrency. Keep one hour beyond the
    # recipe's 12-hour Slurm limit for checkout, staging, and artifact upload.
    timeout-minutes: ${{ inputs.scenario-type == 'agentic-coding' && 780 || 480 }}
    name: >-
      ${{ inputs.klaud-run && 'klaud | ' || '' }}p${{ inputs.priority }} | ${{ fromJSON(inputs.config).model-prefix }} ${{ fromJSON(inputs.config).precision }} ${{ inputs.runner }} ${{ fromJSON(inputs.config).framework == 'sglang' && 'sgl' || fromJSON(inputs.config).framework == 'dynamo-sglang' && 'dyn-sgl' || fromJSON(inputs.config).framework == 'sglang-disagg' && 'sgl-disagg' || fromJSON(inputs.config).framework }}
      ${{ format('{0}', fromJSON(inputs.config).disagg) == 'true' && format('{0}P ', fromJSON(inputs.config).prefill.num-worker) || '' }}(TP${{ fromJSON(inputs.config).prefill.tp }}${{ format('{0}', fromJSON(inputs.config).prefill.pp) != '' && format('{0}', fromJSON(inputs.config).prefill.pp) != '1' && format('/PP{0}', fromJSON(inputs.config).prefill.pp) || '' }}${{ format('{0}', fromJSON(inputs.config).prefill.dcp-size) != '' && format('{0}', fromJSON(inputs.config).prefill.dcp-size) != '1' && format('/DCP{0}', fromJSON(inputs.config).prefill.dcp-size) || '' }}${{ format('{0}', fromJSON(inputs.config).prefill.pcp-size) != '' && format('{0}', fromJSON(inputs.config).prefill.pcp-size) != '1' && format('/PCP{0}', fromJSON(inputs.config).prefill.pcp-size) || '' }}${{ format('{0}', fromJSON(inputs.config).prefill.ep) != '' && format('{0}', fromJSON(inputs.config).prefill.ep) != '1' && format('/EP{0}', fromJSON(inputs.config).prefill.ep) || '' }}${{ format('{0}', fromJSON(inputs.config).prefill.dp-attn) == 'true' && '/DPA' || '' }})
      ${{ format('{0}', fromJSON(inputs.config).disagg) == 'true' && format('x {0}D (TP{1}{2}{3}{4}{5}{6})', fromJSON(inputs.config).decode.num-worker, fromJSON(inputs.config).decode.tp, format('{0}', fromJSON(inputs.config).decode.pp) != '' && format('{0}', fromJSON(inputs.config).decode.pp) != '1' && format('/PP{0}', fromJSON(inputs.config).decode.pp) || '', format('{0}', fromJSON(inputs.config).decode.dcp-size) != '' && format('{0}', fromJSON(inputs.config).decode.dcp-size) != '1' && format('/DCP{0}', fromJSON(inputs.config).decode.dcp-size) || '', format('{0}', fromJSON(inputs.config).decode.pcp-size) != '' && format('{0}', fromJSON(inputs.config).decode.pcp-size) != '1' && format('/PCP{0}', fromJSON(inputs.config).decode.pcp-size) || '', format('{0}', fromJSON(inputs.config).decode.ep) != '' && format('{0}', fromJSON(inputs.config).decode.ep) != '1' && format('/EP{0}', fromJSON(inputs.config).decode.ep) || '', format('{0}', fromJSON(inputs.config).decode.dp-attn) == 'true' && '/DPA' || '') || '' }}
      ${{ fromJSON(inputs.config).spec-decoding != 'none' && fromJSON(inputs.config).spec-decoding || '' }}
      ${{ inputs.scenario-type == 'agentic-coding' && fromJSON(inputs.config).kv-offloading != '' && fromJSON(inputs.config).kv-offloading != 'none' && format('{0} KV offload', fromJSON(inputs.config).kv-offloading) || '' }}
      ${{ inputs.scenario-type == 'agentic-coding' && fromJSON(inputs.config)['kv-offload-backend'].name != '' && fromJSON(inputs.config)['kv-offload-backend'].name != 'none' && fromJSON(inputs.config)['kv-offload-backend'].name != 'default' && fromJSON(inputs.config)['kv-offload-backend'].name || '' }}
      c${{ join(fromJson(inputs.conc-list), 'x') }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }}

    steps:
      - name: Slurm cleanup (pre-run)
        run: &slurm-cleanup |
          if command -v squeue >/dev/null 2>&1; then
            for job_name in "${RUNNER_NAME}" "inferencex-${RUNNER_NAME}"; do
              echo "[Slurm] Cleaning up jobs named: $job_name ..."
              scancel --user="$USER" --name="$job_name" || true
              while [ -n "$(squeue --user="$USER" --name="$job_name" --noheader --format='%i')" ]; do
                squeue --user="$USER" --name="$job_name"
                sleep 5
              done
            done
          fi

      # A cancelled or crashed job can leave git state in this persistent
      # workspace that breaks checkout; re-cloning fixes it and is idempotent.
      - id: checkout
        continue-on-error: true
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with: &checkout-inputs
          fetch-depth: 1
          ref: ${{ inputs.ref || github.sha }}
          clean: true
          submodules: true
          persist-credentials: false

      - name: Wipe workspace after failed checkout
        if: ${{ steps.checkout.outcome == 'failure' }}
        run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} +

      - name: Retry checkout
        if: ${{ steps.checkout.outcome == 'failure' }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with: *checkout-inputs

      - name: Checkout result tooling
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ github.workflow_sha }}
          path: .result-tooling
          # Only inferencex-e2e/ is read from this checkout.
          sparse-checkout: inferencex-e2e
          persist-credentials: false

      - name: Set up uv
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          enable-cache: false

      - name: Prepare result-processing Python
        env: &uv-setup-env
          UV_CACHE_DIR: ${{ github.workspace }}/../.infx-uv-cache
          UV_PYTHON_INSTALL_DIR: ${{ github.workspace }}/../.infx-uv-python
        run: |
          uv python install --quiet --no-bin 3.12
          result_python=$(uv python find --system 3.12)
          "$result_python" --version
          echo "INFERENCEX_RESULTS_PYTHON=$result_python" >> "$GITHUB_ENV"

      - name: Prepare launcher Python
        env: *uv-setup-env
        run: |
          launch_venv="$RUNNER_TEMP/infx-launch-venv"
          uv venv --quiet --python 3.12 "$launch_venv"
          uv pip install --quiet --exclude-newer PT12H --python "$launch_venv/bin/python" \
            -r "$GITHUB_WORKSPACE/.result-tooling/inferencex-e2e/pyproject.toml"
          echo "INFERENCEX_LAUNCH_PYTHON=$launch_venv/bin/python" >> "$GITHUB_ENV"

      - name: Launcher cleanup (pre-run)
        env:
          PYTHONPATH: ${{ github.workspace }}/.result-tooling/inferencex-e2e
        run: &launcher-cleanup |
          "$INFERENCEX_LAUNCH_PYTHON" -P -m infx.launch cleanup

      - name: Launch multi-node job script
        env:
          PREFILL_ADDITIONAL_SETTINGS: ${{ toJSON(fromJSON(inputs.config).prefill.additional-settings) }}
          DECODE_ADDITIONAL_SETTINGS: ${{ toJSON(fromJSON(inputs.config).decode.additional-settings) }}
          RUNNER_NAME: ${{ runner.name }}
          RUNNER_TYPE: ${{ inputs.runner }}
          RESULT_FILENAME_BASE: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_prefill-tp${{ env.PREFILL_TP }}-pp${{ env.PREFILL_PP_SIZE }}-dcp${{ env.PREFILL_DCP_SIZE }}-pcp${{ env.PREFILL_PCP_SIZE }}${{ env.DWDP_ARTIFACT_SUFFIX }}-ep${{ env.PREFILL_EP }}-dp${{ env.PREFILL_DP_ATTN }}-nw${{ env.PREFILL_NUM_WORKERS }}_decode-tp${{ env.DECODE_TP }}-pp${{ env.DECODE_PP_SIZE }}-dcp${{ env.DECODE_DCP_SIZE }}-pcp${{ env.DECODE_PCP_SIZE }}-ep${{ env.DECODE_EP }}-dp${{ env.DECODE_DP_ATTN }}-nw${{ env.DECODE_NUM_WORKERS }}_disagg-${{ env.DISAGG }}_spec-${{ env.SPEC_DECODING }}_conc${{ join(fromJson(inputs.conc-list), 'x') }}_${{ runner.name }}
        run: |
          if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
          export GITHUB_WORKSPACE="$PWD"
          echo "INFERENCEX_E2E_ROOT=$PWD" >> "$GITHUB_ENV"
          set -x
          RESULT_FILENAME=$(python3 -m infx.results.result_filename)
          export RESULT_FILENAME
          # Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled
          echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV"
          echo "EVAL_ARTIFACT_RECIPE=${RECIPE_FINGERPRINT:0:16}" >> "$GITHUB_ENV"
          eval_artifact_conc="$(python3 -c 'import hashlib,os; print(hashlib.sha256(os.environ["CONC_LIST"].encode()).hexdigest()[:12])')"
          echo "EVAL_ARTIFACT_CONC=${eval_artifact_conc}" >> "$GITHUB_ENV"

          source benchmarks/runtime_settings.sh
          source benchmarks/multi_node/runtime_settings.sh
          # Assign data literally; recipe values must never become shell code.
          settings_json=$(jq -cen \
            --argjson prefill "$PREFILL_ADDITIONAL_SETTINGS" \
            --argjson decode "$DECODE_ADDITIONAL_SETTINGS" \
            '($prefill // []) + ($decode // []) |
             if all(.[]; type == "string" and test("^[A-Za-z_][A-Za-z0-9_]*=")) then .
             else error("additional-settings must contain NAME=value assignments") end')
          while IFS= read -r -d '' setting; do
            export "$setting"
          done < <(jq -j '.[] + "\u0000"' <<< "$settings_json")
          # Resolve the workflow's documented automatic eval-concurrency selection.
          if [[ -z "$EVAL_CONC" ]]; then
            EVAL_CONC=$(python3 -c 'import os; print(max(map(int, os.environ["CONC_LIST"].split())))')
          fi
          export EVAL_CONC
          export IS_MULTINODE=true
          "$INFERENCEX_LAUNCH_PYTHON" -m infx.launch run
          if [ "${EVAL_ONLY}" = "true" ]; then
            echo "Eval-only mode: skipping benchmark result file check"
            # Verify eval produced results
            if ! ls results*.json 1>/dev/null 2>&1; then
              echo "Eval-only run failed: no results*.json files found." >&2
              exit 1
            fi
          elif [ "${SCENARIO_TYPE}" = "agentic-coding" ]; then
            expected_count=$(wc -w <<< "$CONC_LIST" | tr -d ' ')
            shopt -s nullglob
            agentic_results=("${RESULT_FILENAME}"_conc*.json)
            shopt -u nullglob
            if [ "${#agentic_results[@]}" -ne "$expected_count" ]; then
              echo "Run failed: expected $expected_count agentic results, found ${#agentic_results[@]}." >&2
              exit 1
            fi
            # Existence is not enough: the agentic aggregation step writes the
            # aggregate even when aiperf recorded zero valid requests. Require
            # successful requests in every concurrency result.
            for result_file in "${agentic_results[@]}"; do
              echo "Found agentic result file: $result_file"
              ok=$(python3 -c "import json,sys; d=json.load(open(sys.argv[1])); print(int(bool(d.get('num_requests_successful'))))" "$result_file" 2>/dev/null || echo 0)
              if [ "$ok" != "1" ]; then
                echo "Run failed: $result_file has zero successful requests." >&2
                exit 1
              fi
            done
          else
            if ls -- "${RESULT_FILENAME}"_*.json 1> /dev/null 2>&1; then
              printf 'Found result file: %s\n' "${RESULT_FILENAME}"_*.json
            else
              echo "Run failed: No benchmark result files found for ${RESULT_FILENAME}_*.json" >&2
              exit 1
            fi
          fi

      - name: Process result
        if: ${{ always() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }}
        env:
          RUNNER_TYPE: ${{ inputs.runner }}
          PYTHONPATH: ${{ github.workspace }}/.result-tooling/inferencex-e2e
        run: |
          if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
          source "$PYTHONPATH/benchmarks/check_env.sh"
          check_env_vars INFERENCEX_RESULTS_PYTHON
          # Launcher stamp supplies the producer SHA unless the
          # power-producer-sha input is set as a manual override
          if [ -z "$POWER_PRODUCER_SHA" ] && [ -f power-producer-sha.txt ]; then
            export POWER_PRODUCER_SHA="$(cat power-producer-sha.txt)"
            echo "POWER_PRODUCER_SHA derived from launcher stamp: $POWER_PRODUCER_SHA"
          fi
          "$INFERENCEX_RESULTS_PYTHON" -P -m infx.results.fixed_sequence --all

      - name: Upload result
        id: upload-result
        if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bmk_${{ env.RESULT_FILENAME }}
          path: ${{ env.INFERENCEX_E2E_ROOT }}/agg_${{ env.RESULT_FILENAME }}_*.json

      - name: Wait before retrying upload result
        if: ${{ always() && steps.upload-result.outcome == 'failure' }}
        run: sleep 10

      - name: Retry upload result (attempt 2)
        id: retry-upload-result
        if: ${{ always() && steps.upload-result.outcome == 'failure' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: bmk_${{ env.RESULT_FILENAME }}
          path: ${{ env.INFERENCEX_E2E_ROOT }}/agg_${{ env.RESULT_FILENAME }}_*.json

      - name: Wait before final result upload attempt
        if: ${{ always() && steps.retry-upload-result.outcome == 'failure' }}
        run: sleep 30

      - name: Retry upload result (attempt 3)
        if: ${{ always() && steps.retry-upload-result.outcome == 'failure' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: bmk_${{ env.RESULT_FILENAME }}
          path: ${{ env.INFERENCEX_E2E_ROOT }}/agg_${{ env.RESULT_FILENAME }}_*.json

      - name: Upload power audit bundle
        if: ${{ always() && !inputs.eval-only }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: power_audit_${{ env.RESULT_FILENAME }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/${{ env.RESULT_FILENAME }}_*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/agg_${{ env.RESULT_FILENAME }}_*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/power_validation_${{ env.RESULT_FILENAME }}_*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/LOGS/power/**
            ${{ env.INFERENCEX_E2E_ROOT }}/result_processing_${{ env.RESULT_FILENAME }}.json
            ${{ env.INFERENCEX_E2E_ROOT }}/LOGS/*/results_*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/LOGS/agentic/**/agentic_power_concurrency_*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/LOGS/agentic/**/agentic_power_timezone_offset.txt
            ${{ env.INFERENCEX_E2E_ROOT }}/LOGS/agentic/**/power_validation.json
          if-no-files-found: ignore

      - name: Upload server logs
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: multinode_server_logs_${{ env.RESULT_FILENAME }}${{ inputs.eval-only && format('_{0}_{1}', env.EVAL_FRAMEWORK, env.EVAL_SUITE) || '' }}
          # multinode launchers package server logs into this tarball.
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/multinode_server_logs.tar.gz
            ${{ env.INFERENCEX_E2E_ROOT }}/srt-setup.log
          if-no-files-found: ignore

      - name: Upload agentic aggregated result
        id: upload-agentic-aggregated
        if: ${{ always() && !inputs.eval-only && inputs.scenario-type == 'agentic-coding' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bmk_agentic_${{ env.RESULT_FILENAME }}
          path: ${{ env.INFERENCEX_E2E_ROOT }}/${{ env.RESULT_FILENAME }}_conc*.json

      - name: Wait before retrying upload agentic aggregated result
        if: ${{ always() && steps.upload-agentic-aggregated.outcome == 'failure' }}
        run: sleep 10

      - name: Retry upload agentic aggregated result (attempt 2)
        id: retry-upload-agentic-aggregated
        if: ${{ always() && steps.upload-agentic-aggregated.outcome == 'failure' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: bmk_agentic_${{ env.RESULT_FILENAME }}
          path: ${{ env.INFERENCEX_E2E_ROOT }}/${{ env.RESULT_FILENAME }}_conc*.json

      - name: Wait before final agentic aggregated result
        if: ${{ always() && steps.retry-upload-agentic-aggregated.outcome == 'failure' }}
        run: sleep 30

      - name: Retry upload agentic aggregated result (attempt 3)
        if: ${{ always() && steps.retry-upload-agentic-aggregated.outcome == 'failure' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: bmk_agentic_${{ env.RESULT_FILENAME }}
          path: ${{ env.INFERENCEX_E2E_ROOT }}/${{ env.RESULT_FILENAME }}_conc*.json

      - name: Upload agentic raw results
        id: upload-agentic-raw
        if: ${{ always() && inputs.scenario-type == 'agentic-coding' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: agentic_${{ env.RESULT_FILENAME }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/LOGS/agentic/**
            !${{ env.INFERENCEX_E2E_ROOT }}/LOGS/agentic/**/aiperf_artifacts/inputs.json
            !${{ env.INFERENCEX_E2E_ROOT }}/LOGS/agentic/**/aiperf_artifacts/profile_export_raw.jsonl
          if-no-files-found: ignore

      - name: Wait before retrying upload agentic raw results
        if: ${{ always() && steps.upload-agentic-raw.outcome == 'failure' }}
        run: sleep 10

      - name: Retry upload agentic raw results (attempt 2)
        id: retry-upload-agentic-raw
        if: ${{ always() && steps.upload-agentic-raw.outcome == 'failure' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: agentic_${{ env.RESULT_FILENAME }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/LOGS/agentic/**
            !${{ env.INFERENCEX_E2E_ROOT }}/LOGS/agentic/**/aiperf_artifacts/inputs.json
            !${{ env.INFERENCEX_E2E_ROOT }}/LOGS/agentic/**/aiperf_artifacts/profile_export_raw.jsonl
          if-no-files-found: ignore

      - name: Wait before final agentic raw results
        if: ${{ always() && steps.retry-upload-agentic-raw.outcome == 'failure' }}
        run: sleep 30

      - name: Retry upload agentic raw results (attempt 3)
        if: ${{ always() && steps.retry-upload-agentic-raw.outcome == 'failure' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: agentic_${{ env.RESULT_FILENAME }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/LOGS/agentic/**
            !${{ env.INFERENCEX_E2E_ROOT }}/LOGS/agentic/**/aiperf_artifacts/inputs.json
            !${{ env.INFERENCEX_E2E_ROOT }}/LOGS/agentic/**/aiperf_artifacts/profile_export_raw.jsonl
          if-no-files-found: ignore

      - name: Upload eval results (if any)
        id: upload-eval
        if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}${{ env.DWDP_ARTIFACT_SUFFIX }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_g${{ env.DISAGG }}_r${{ env.EVAL_ARTIFACT_RECIPE }}_c${{ env.EVAL_ARTIFACT_CONC }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ runner.name }}_${{ github.run_attempt }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/meta_env.json
            ${{ env.INFERENCEX_E2E_ROOT }}/results*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_report.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_results.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/sample*.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/agent_preds.json
            ${{ env.INFERENCEX_E2E_ROOT }}/predictions.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/swebench_report_*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_artifacts.tar.gz
            ${{ env.INFERENCEX_E2E_ROOT }}/*.traj*
          if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }}

      - name: Wait before retrying upload eval results (if any)
        if: ${{ always() && steps.upload-eval.outcome == 'failure' }}
        run: sleep 10

      - name: Retry upload eval results (attempt 2)
        id: retry-upload-eval
        if: ${{ always() && steps.upload-eval.outcome == 'failure' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}${{ env.DWDP_ARTIFACT_SUFFIX }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_g${{ env.DISAGG }}_r${{ env.EVAL_ARTIFACT_RECIPE }}_c${{ env.EVAL_ARTIFACT_CONC }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ runner.name }}_${{ github.run_attempt }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/meta_env.json
            ${{ env.INFERENCEX_E2E_ROOT }}/results*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_report.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_results.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/sample*.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/agent_preds.json
            ${{ env.INFERENCEX_E2E_ROOT }}/predictions.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/swebench_report_*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_artifacts.tar.gz
            ${{ env.INFERENCEX_E2E_ROOT }}/*.traj*
          if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }}

      - name: Wait before final eval upload attempt
        if: ${{ always() && steps.retry-upload-eval.outcome == 'failure' }}
        run: sleep 30

      - name: Retry upload eval results (attempt 3)
        if: ${{ always() && steps.retry-upload-eval.outcome == 'failure' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}${{ env.DWDP_ARTIFACT_SUFFIX }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_g${{ env.DISAGG }}_r${{ env.EVAL_ARTIFACT_RECIPE }}_c${{ env.EVAL_ARTIFACT_CONC }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ runner.name }}_${{ github.run_attempt }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/meta_env.json
            ${{ env.INFERENCEX_E2E_ROOT }}/results*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_report.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_results.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/sample*.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/*_artifacts.tar.gz
          if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }}

      - name: Verify eval scores
        if: ${{ (success() || failure()) && inputs.eval-only }}
        env:
          PYTHONPATH: ${{ github.workspace }}/.result-tooling/inferencex-e2e
        run: |
          if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
          source "$PYTHONPATH/benchmarks/check_env.sh"
          check_env_vars INFERENCEX_RESULTS_PYTHON
          expected_concs="${EVAL_CONC}"
          if [[ -z "${expected_concs}" ]]; then
            expected_concs="$(printf '%s\n' "${CONC_LIST}" | tr ' ' '\n' | sort -n | tail -1)"
          fi
          "$INFERENCEX_RESULTS_PYTHON" -P -m infx.evals.validate_scores --expected-concs "${expected_concs}"

      - name: Cleanup eval outputs (post-upload)
        if: ${{ always() && (inputs.run-eval || inputs.eval-only) }}
        run: |
          if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
          rm -f meta_env.json || true
          rm -f results*.json || true
          rm -f -- ./*_report.json || true
          rm -f -- ./*_results.jsonl || true
          rm -f -- ./*_artifacts.tar.gz || true
          rm -f sample*.jsonl || true

      - name: Slurm cleanup (post-run)
        if: always()
        run: *slurm-cleanup

      - name: Launcher cleanup (post-run)
        if: ${{ always() && env.INFERENCEX_LAUNCH_PYTHON != '' }}
        env:
          PYTHONPATH: ${{ github.workspace }}/.result-tooling/inferencex-e2e
        run: *launcher-cleanup