benchmark-tmpl.yml

name: Template - Benchmark
on:
  workflow_call:
    secrets:
      INFERENCEX_OFFICIAL_RO_HF_TOKEN:
        required: true
    inputs:
      config:
        description: "Benchmark configuration as JSON"
        required: true
        type: string
      klaud-run:
        description: "Run Klaud Cold benchmarks at background priority"
        required: false
        type: boolean
        default: false
      runner:
        required: true
        type: string
      priority:
        description: "Higher-is-sooner CI priority score"
        required: true
        type: string
      queue-token:
        description: "One-shot queue token generated for this job"
        required: true
        type: string
      dp-attn:
        required: true
        type: boolean
      run-eval:
        type: boolean
        required: true
      eval-only:
        description: "Run only evals (skip throughput benchmark)"
        type: boolean
        required: false
        default: false
      eval-framework:
        description: "Eval runner (lm-eval, kimi-vendor, minimax-vendor, or bfcl)"
        type: string
        required: false
        default: "lm-eval"
      eval-suite:
        description: "Eval suite (kimi_tool_call_schema, kimi_tool_call_schema_full, minimax_m3_smoke, minimax_m3_full, bfcl_smoke, bfcl_vllm_minimax_m3, or bfcl_vllm_kimi); empty for lm-eval"
        type: string
        required: false
        default: ""
      random-range-ratio:
        required: false
        type: string
        default: '0.8'
      ref:
        description: "Git ref (branch/sha) to checkout"
        required: false
        type: string
      # Agentic-coding inputs. Fixed-seq-len jobs leave these empty.
      scenario-type:
        description: "Scenario type (fixed-seq-len or agentic-coding)"
        required: false
        type: string
        default: 'fixed-seq-len'
      duration:
        description: "Benchmark duration in seconds"
        required: false
        type: string
        default: '3600'
      agentx-fast:
        description: "Use one warmup request per AgentX lane and a 20-minute profile"
        required: false
        type: boolean
        default: false
      eval-limit:
        description: "Eval instance count: empty = whole split (default); N = first-N smoke slice"
        required: false
        type: string
        default: ""
env:
  PYTHONPATH: ${{ github.workspace }}/inferencex-e2e
  INFERENCEX_E2E_ROOT: ${{ github.workspace }}
  PORT: '8888'
  INFMAX_CONTAINER_WORKSPACE: '/workspace'
  DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE: '1'
  MODEL_DOWNLOAD_LOCK_TIMEOUT: '21600'
  ENROOT_IMPORT_TIME_LIMIT: '120'
  SALLOC_TIME_LIMIT: '480'
  KEEP_LOGS: '0'
  IS_MULTINODE: 'false'
  RANDOM_RANGE_RATIO: 0.8
  HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
  HF_HUB_CACHE: '/mnt/hf_hub_cache/'
  EXP_NAME: ${{ fromJSON(inputs.config).exp-name }}
  RECIPE_FINGERPRINT: ${{ fromJSON(inputs.config).recipe-fingerprint || '' }}
  SRT_RECIPE: ${{ fromJSON(inputs.config).srt-recipe || '' }}
  MODEL: ${{ fromJSON(inputs.config).model }}
  THINKING_MODE: thinking_on
  MODEL_PREFIX: ${{ fromJSON(inputs.config).model-prefix }}
  ISL: ${{ inputs.scenario-type == 'agentic-coding' && '0' || fromJSON(inputs.config).isl }}
  OSL: ${{ inputs.scenario-type == 'agentic-coding' && '0' || fromJSON(inputs.config).osl }}
  MAX_MODEL_LEN: ${{ inputs.scenario-type == 'agentic-coding' && '0' || fromJSON(inputs.config).max-model-len }}
  IMAGE: ${{ fromJSON(inputs.config).image }}
  FRAMEWORK: ${{ fromJSON(inputs.config).framework }}
  PRECISION: ${{ fromJSON(inputs.config).precision }}
  TP: ${{ fromJSON(inputs.config).tp }}
  PP_SIZE: ${{ fromJSON(inputs.config).pp }}
  DCP_SIZE: ${{ fromJSON(inputs.config).dcp-size }}
  PCP_SIZE: ${{ fromJSON(inputs.config).pcp-size }}
  EP_SIZE: ${{ fromJSON(inputs.config).ep }}
  DP_ATTENTION: ${{ inputs.dp-attn }}
  CONC: ${{ fromJSON(inputs.config).conc }}
  SPEC_DECODING: ${{ fromJSON(inputs.config).spec-decoding }}
  DISAGG: ${{ inputs.scenario-type == 'agentic-coding' && 'false' || fromJSON(inputs.config).disagg }}
  RUN_EVAL: ${{ inputs.run-eval }}
  EVAL_ONLY: ${{ inputs.eval-only }}
  EVAL_FRAMEWORK: ${{ inputs.eval-framework }}
  EVAL_SUITE: ${{ inputs.eval-suite }}
  # Agentic-coding env. Fixed-seq-len jobs leave these empty.
  SCENARIO_TYPE: ${{ inputs.scenario-type }}
  SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }}
  IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }}
  KV_OFFLOADING: ${{ inputs.scenario-type == 'agentic-coding' && format('{0}', fromJSON(inputs.config).kv-offloading) || '' }}
  KV_OFFLOAD_BACKEND: ${{ inputs.scenario-type == 'agentic-coding' && format('{0}', fromJSON(inputs.config).kv-offload-backend.name) || '' }}
  KV_OFFLOAD_BACKEND_METADATA: ${{ inputs.scenario-type == 'agentic-coding' && fromJSON(inputs.config).kv-offload-backend && toJSON(fromJSON(inputs.config).kv-offload-backend) || '' }}
  ROUTER_METADATA: ${{ fromJSON(inputs.config).router && toJSON(fromJSON(inputs.config).router) || '' }}
  KV_P2P_TRANSFER: ${{ fromJSON(inputs.config).kv-p2p-transfer || '' }}
  TOTAL_CPU_DRAM_GB: ${{ inputs.scenario-type != 'agentic-coding' && '0' || fromJSON(inputs.config).total-cpu-dram-gb }}
  DURATION: ${{ inputs.duration }}
  AIPERF_EXPERIMENTAL_FAST: ${{ inputs.agentx-fast && '1' || '0' }}
  EVAL_LIMIT: ${{ inputs.eval-limit }}
  AIPERF_FAILED_REQUEST_THRESHOLD: '0.10'
  RESULT_DIR: /workspace/results
  PYTHONDONTWRITEBYTECODE: '1'
  PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache

permissions:
  contents: read

jobs:
  benchmark:
    runs-on: >-
      ${{ fromJSON(
        vars.PRIORITY_SCHEDULER_ENABLED == 'true' &&
        (
          vars.NODE_SLOT_SCHEDULER_ENABLED == 'true' &&
          format(
            '["self-hosted",{0},{1},{2},{3}]',
            toJSON(inputs.runner),
            '"nodes:1"',
            toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)),
            toJSON(format('ci-attempt-{0}', github.run_attempt))
          ) ||
          format(
            '["self-hosted",{0},{1},{2}]',
            toJSON(inputs.runner),
            toJSON(format('ci-job-{0}-{1}', inputs.priority, inputs.queue-token)),
            toJSON(format('ci-attempt-{0}', github.run_attempt))
          )
        ) ||
        format('[{0}]', toJSON(inputs.runner))
      ) }}
    timeout-minutes: ${{ inputs.runner == 'cluster:h200-dgxc' && fromJSON(inputs.config).model-prefix == 'dsv41flash' && fromJSON(inputs.config).framework == 'sglang' && inputs.scenario-type == 'agentic-coding' && !inputs.eval-only && fromJSON(inputs.config).conc >= 64 && 1470 || 500 }}
    name: >-
      ${{ inputs.klaud-run && 'klaud | ' || '' }}p${{ inputs.priority }} | ${{ fromJSON(inputs.config).model-prefix }} ${{ fromJSON(inputs.config).precision }} ${{ inputs.runner }} ${{ fromJSON(inputs.config).framework == 'sglang' && 'sgl' || fromJSON(inputs.config).framework == 'dynamo-sglang' && 'dyn-sgl' || fromJSON(inputs.config).framework == 'sglang-disagg' && 'sgl-disagg' || fromJSON(inputs.config).framework }}
      TP${{ fromJSON(inputs.config).tp }}${{ format('{0}', fromJSON(inputs.config).pp) != '' && format('{0}', fromJSON(inputs.config).pp) != '1' && format('/PP{0}', fromJSON(inputs.config).pp) || '' }}${{ format('{0}', fromJSON(inputs.config).dcp-size) != '' && format('{0}', fromJSON(inputs.config).dcp-size) != '1' && format('/DCP{0}', fromJSON(inputs.config).dcp-size) || '' }}${{ format('{0}', fromJSON(inputs.config).pcp-size) != '' && format('{0}', fromJSON(inputs.config).pcp-size) != '1' && format('/PCP{0}', fromJSON(inputs.config).pcp-size) || '' }}${{ format('{0}', fromJSON(inputs.config).ep) != '' && format('{0}', fromJSON(inputs.config).ep) != '1' && format('/EP{0}', fromJSON(inputs.config).ep) || '' }}${{ inputs.dp-attn && '/DPA' || '' }}
      ${{ fromJSON(inputs.config).spec-decoding != 'none' && fromJSON(inputs.config).spec-decoding || '' }}
      ${{ inputs.scenario-type == 'agentic-coding' && format('{0}', fromJSON(inputs.config).kv-offloading) != '' && fromJSON(inputs.config).kv-offloading != 'none' && format('{0} KV offload', fromJSON(inputs.config).kv-offloading) || '' }}
      ${{ inputs.scenario-type == 'agentic-coding' && fromJSON(inputs.config).kv-offload-backend.name != 'none' && fromJSON(inputs.config).kv-offload-backend.name != 'default' && format('{0}', fromJSON(inputs.config).kv-offload-backend.name) || '' }}
      c${{ fromJSON(inputs.config).conc }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }}
    steps:
      - name: Repair root-owned artifacts (pre-run)
        if: ${{ inputs.runner == 'cluster:mi355x-amds' }}
        run: |
          if [[ -d "$GITHUB_WORKSPACE" ]]; then
            sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
          fi

      - name: Slurm cleanup (pre-run)
        run: &slurm-cleanup |
          if command -v squeue >/dev/null 2>&1; then
            for job_name in "${RUNNER_NAME}" "inferencex-${RUNNER_NAME}"; do
              echo "[Slurm] Cleaning up jobs with name: ${job_name} ..."
              scancel --name="${job_name}" || true
              while [ -n "$(squeue --name="${job_name}" --noheader --format='%i')" ]; do
                squeue --name="${job_name}"
                sleep 5
              done
            done
          fi

      # A cancelled or crashed job can leave git state in this persistent
      # workspace that breaks checkout; re-cloning fixes it and is idempotent.
      - id: checkout
        continue-on-error: true
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with: &checkout-inputs
          fetch-depth: 1
          ref: ${{ inputs.ref || github.sha }}
          clean: true
          submodules: true
          persist-credentials: false

      - name: Wipe workspace after failed checkout
        if: ${{ steps.checkout.outcome == 'failure' }}
        run: find "$GITHUB_WORKSPACE" -mindepth 1 -maxdepth 1 -exec rm -rf {} +

      - name: Retry checkout
        if: ${{ steps.checkout.outcome == 'failure' }}
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with: *checkout-inputs

      - name: Checkout result tooling
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ github.workflow_sha }}
          path: .result-tooling
          # Only inferencex-e2e/ is read from this checkout.
          sparse-checkout: inferencex-e2e
          persist-credentials: false

      - name: Set up uv
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          enable-cache: false

      - name: Prepare result-processing Python
        env: &uv-setup-env
          UV_CACHE_DIR: ${{ github.workspace }}/../.infx-uv-cache
          UV_PYTHON_INSTALL_DIR: ${{ github.workspace }}/../.infx-uv-python
        run: |
          uv python install --quiet --no-bin 3.12
          result_python=$(uv python find --system 3.12)
          "$result_python" --version
          echo "INFERENCEX_RESULTS_PYTHON=$result_python" >> "$GITHUB_ENV"

      - name: Prepare launcher Python
        env: *uv-setup-env
        run: |
          launch_venv="$RUNNER_TEMP/infx-launch-venv"
          uv venv --quiet --python 3.12 "$launch_venv"
          uv pip install --quiet --exclude-newer PT12H --python "$launch_venv/bin/python" \
            -r "$GITHUB_WORKSPACE/.result-tooling/inferencex-e2e/pyproject.toml"
          echo "INFERENCEX_LAUNCH_PYTHON=$launch_venv/bin/python" >> "$GITHUB_ENV"

      - name: Launcher cleanup (pre-run)
        env:
          PYTHONPATH: ${{ github.workspace }}/.result-tooling/inferencex-e2e
        run: &launcher-cleanup |
          "$INFERENCEX_LAUNCH_PYTHON" -P -m infx.launch cleanup

      - name: Launch job script
        env:
          RUNNER_NAME: ${{ runner.name }}
          RUNNER_TYPE: ${{ inputs.runner }}
          RESULT_FILENAME_BASE: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_tp${{ env.TP }}-pp${{ env.PP_SIZE }}-dcp${{ env.DCP_SIZE }}-pcp${{ env.PCP_SIZE }}-ep${{ env.EP_SIZE }}-dpa${{ env.DP_ATTENTION }}_disagg-${{ env.DISAGG }}_spec-${{ env.SPEC_DECODING }}_conc${{ env.CONC }}_${{ runner.name }}
          # Suppress per-job eval markdown from being appended to the step summary.
          # We'll publish a single combined eval table in the collection job instead.
          GITHUB_STEP_SUMMARY: ''
        run: |
          if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
          export GITHUB_WORKSPACE="$PWD"
          echo "INFERENCEX_E2E_ROOT=$PWD" >> "$GITHUB_ENV"
          RESULT_FILENAME=$(python3 -m infx.results.result_filename)
          export RESULT_FILENAME
          export GPU_COUNT=$((TP * PP_SIZE * PCP_SIZE))
          echo "GPU_COUNT=${GPU_COUNT}" >> "$GITHUB_ENV"

          # Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled
          echo "RESULT_FILENAME=${RESULT_FILENAME}" >> "$GITHUB_ENV"

          source benchmarks/runtime_settings.sh
          "$INFERENCEX_LAUNCH_PYTHON" -m infx.launch run

          if [ "${EVAL_ONLY}" = "true" ]; then
            echo "Eval-only mode: skipping benchmark result file check"
            # Verify eval produced results
            if ! ls results*.json 1>/dev/null 2>&1; then
              echo "Eval-only run failed: no results*.json files found." >&2
              exit 1
            fi
          else
            FOUND_RESULT_FILE=
            for i in {1..10}; do
              if [ -f "$RESULT_FILENAME.json" ]; then
                FOUND_RESULT_FILE=true
                break
              fi
              echo "Waiting for result file... (attempt $i)"
              sleep 1
            done

            if [ -z "$FOUND_RESULT_FILE" ]; then
              echo "Run failed: Benchmark result $RESULT_FILENAME.json not found." >&2
              exit 1
            fi
          fi

      - name: Validate agentic result
        if: ${{ !inputs.eval-only && inputs.scenario-type == 'agentic-coding' }}
        env:
          PYTHONPATH: ${{ github.workspace }}/.result-tooling/inferencex-e2e
        run: |
          if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
          source "$PYTHONPATH/benchmarks/check_env.sh"
          check_env_vars INFERENCEX_RESULTS_PYTHON AIPERF_FAILED_REQUEST_THRESHOLD
          "$INFERENCEX_RESULTS_PYTHON" -P -m infx.results.agentic.validate_agentic_result \
            results/aiperf_artifacts \
            --failed-request-threshold "$AIPERF_FAILED_REQUEST_THRESHOLD"

      - name: Process result
        if: ${{ always() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }}
        env:
          RUNNER_TYPE: ${{ inputs.runner }}
          PYTHONPATH: ${{ github.workspace }}/.result-tooling/inferencex-e2e
        run: |
          if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
          source "$PYTHONPATH/benchmarks/check_env.sh"
          check_env_vars INFERENCEX_RESULTS_PYTHON
          if [ ! -f "$RESULT_FILENAME.json" ]; then
            echo "no raw result to process: $RESULT_FILENAME.json" >&2
            exit 1
          fi
          "$INFERENCEX_RESULTS_PYTHON" -P -m infx.results.fixed_sequence

      - name: Upload result
        id: upload-result
        if: ${{ success() && env.RESULT_FILENAME != '' && !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bmk_${{ env.RESULT_FILENAME }}
          path: ${{ env.INFERENCEX_E2E_ROOT }}/agg_${{ env.RESULT_FILENAME }}.json

      - name: Wait before retrying upload result
        if: ${{ always() && steps.upload-result.outcome == 'failure' }}
        run: sleep 10

      - name: Retry upload result (attempt 2)
        id: retry-upload-result
        if: ${{ always() && steps.upload-result.outcome == 'failure' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: bmk_${{ env.RESULT_FILENAME }}
          path: ${{ env.INFERENCEX_E2E_ROOT }}/agg_${{ env.RESULT_FILENAME }}.json

      - name: Wait before final result upload attempt
        if: ${{ always() && steps.retry-upload-result.outcome == 'failure' }}
        run: sleep 30

      - name: Retry upload result (attempt 3)
        if: ${{ always() && steps.retry-upload-result.outcome == 'failure' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: bmk_${{ env.RESULT_FILENAME }}
          path: ${{ env.INFERENCEX_E2E_ROOT }}/agg_${{ env.RESULT_FILENAME }}.json

      - name: Upload agentic aggregated result
        id: upload-agentic-aggregated
        if: ${{ always() && inputs.scenario-type == 'agentic-coding' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bmk_agentic_${{ env.RESULT_FILENAME }}
          path: ${{ env.INFERENCEX_E2E_ROOT }}/${{ env.RESULT_FILENAME }}.json

      - name: Wait before retrying upload agentic aggregated result
        if: ${{ always() && steps.upload-agentic-aggregated.outcome == 'failure' }}
        run: sleep 10

      - name: Retry upload agentic aggregated result (attempt 2)
        id: retry-upload-agentic-aggregated
        if: ${{ always() && steps.upload-agentic-aggregated.outcome == 'failure' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: bmk_agentic_${{ env.RESULT_FILENAME }}
          path: ${{ env.INFERENCEX_E2E_ROOT }}/${{ env.RESULT_FILENAME }}.json

      - name: Wait before final agentic aggregated result
        if: ${{ always() && steps.retry-upload-agentic-aggregated.outcome == 'failure' }}
        run: sleep 30

      - name: Retry upload agentic aggregated result (attempt 3)
        if: ${{ always() && steps.retry-upload-agentic-aggregated.outcome == 'failure' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: bmk_agentic_${{ env.RESULT_FILENAME }}
          path: ${{ env.INFERENCEX_E2E_ROOT }}/${{ env.RESULT_FILENAME }}.json

      - name: Upload agentic raw results
        id: upload-agentic-raw
        if: ${{ always() && inputs.scenario-type == 'agentic-coding' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: agentic_${{ env.RESULT_FILENAME }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/results/**
            !${{ env.INFERENCEX_E2E_ROOT }}/results/aiperf_artifacts/inputs.json
            !${{ env.INFERENCEX_E2E_ROOT }}/results/aiperf_artifacts/profile_export_raw.jsonl
          if-no-files-found: ignore

      - name: Wait before retrying upload agentic raw results
        if: ${{ always() && steps.upload-agentic-raw.outcome == 'failure' }}
        run: sleep 10

      - name: Retry upload agentic raw results (attempt 2)
        id: retry-upload-agentic-raw
        if: ${{ always() && steps.upload-agentic-raw.outcome == 'failure' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: agentic_${{ env.RESULT_FILENAME }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/results/**
            !${{ env.INFERENCEX_E2E_ROOT }}/results/aiperf_artifacts/inputs.json
            !${{ env.INFERENCEX_E2E_ROOT }}/results/aiperf_artifacts/profile_export_raw.jsonl
          if-no-files-found: ignore

      - name: Wait before final agentic raw results
        if: ${{ always() && steps.retry-upload-agentic-raw.outcome == 'failure' }}
        run: sleep 30

      - name: Retry upload agentic raw results (attempt 3)
        if: ${{ always() && steps.retry-upload-agentic-raw.outcome == 'failure' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: agentic_${{ env.RESULT_FILENAME }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/results/**
            !${{ env.INFERENCEX_E2E_ROOT }}/results/aiperf_artifacts/inputs.json
            !${{ env.INFERENCEX_E2E_ROOT }}/results/aiperf_artifacts/profile_export_raw.jsonl
          if-no-files-found: ignore

      - name: Upload server logs
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ inputs.eval-only && 'eval_server_logs_' || 'server_logs_' }}${{ env.RESULT_FILENAME }}${{ inputs.eval-only && format('_{0}_{1}', env.EVAL_FRAMEWORK, env.EVAL_SUITE) || '' }}
          # fixed-seq writes server.log at root; agentic writes logs under results/.
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/server.log
            ${{ env.INFERENCEX_E2E_ROOT }}/results/*.log
            ${{ env.INFERENCEX_E2E_ROOT }}/results/*_config.json
            ${{ env.INFERENCEX_E2E_ROOT }}/srt-single-node-logs.tar.gz
            ${{ env.INFERENCEX_E2E_ROOT }}/srt-single-node-submission.json
            ${{ env.INFERENCEX_E2E_ROOT }}/srt-slurm-sha.txt
            ${{ env.INFERENCEX_E2E_ROOT }}/srt-setup.log
          if-no-files-found: ignore

      - name: Upload eval results (if any)
        id: upload-eval
        if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: eval_${{ env.RESULT_FILENAME }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ github.run_attempt }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/meta_env.json
            ${{ env.INFERENCEX_E2E_ROOT }}/results*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_report.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_results.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/*_artifacts.tar.gz
            ${{ env.INFERENCEX_E2E_ROOT }}/sample*.jsonl
          if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }}

      - name: Wait before retrying upload eval results (if any)
        if: ${{ always() && steps.upload-eval.outcome == 'failure' }}
        run: sleep 10

      - name: Retry upload eval results (attempt 2)
        id: retry-upload-eval
        if: ${{ always() && steps.upload-eval.outcome == 'failure' }}
        continue-on-error: true
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: eval_${{ env.RESULT_FILENAME }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ github.run_attempt }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/meta_env.json
            ${{ env.INFERENCEX_E2E_ROOT }}/results*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_report.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_results.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/*_artifacts.tar.gz
            ${{ env.INFERENCEX_E2E_ROOT }}/sample*.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/agent_preds.json
            ${{ env.INFERENCEX_E2E_ROOT }}/predictions.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/swebench_report_*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*.traj*
          if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }}

      - name: Wait before final eval upload attempt
        if: ${{ always() && steps.retry-upload-eval.outcome == 'failure' }}
        run: sleep 30

      - name: Retry upload eval results (attempt 3)
        if: ${{ always() && steps.retry-upload-eval.outcome == 'failure' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          overwrite: true
          name: eval_${{ env.RESULT_FILENAME }}_${{ env.EVAL_FRAMEWORK }}_${{ env.EVAL_SUITE }}_${{ github.run_attempt }}
          path: |
            ${{ env.INFERENCEX_E2E_ROOT }}/meta_env.json
            ${{ env.INFERENCEX_E2E_ROOT }}/results*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_report.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*_results.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/*_artifacts.tar.gz
            ${{ env.INFERENCEX_E2E_ROOT }}/sample*.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/agent_preds.json
            ${{ env.INFERENCEX_E2E_ROOT }}/predictions.jsonl
            ${{ env.INFERENCEX_E2E_ROOT }}/swebench_report_*.json
            ${{ env.INFERENCEX_E2E_ROOT }}/*.traj*
          if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }}

      - name: Verify eval scores
        if: ${{ (success() || failure()) && inputs.eval-only }}
        env:
          PYTHONPATH: ${{ github.workspace }}/.result-tooling/inferencex-e2e
        run: |
          if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
          source "$PYTHONPATH/benchmarks/check_env.sh"
          check_env_vars INFERENCEX_RESULTS_PYTHON
          "$INFERENCEX_RESULTS_PYTHON" -P -m infx.evals.validate_scores

      - name: Cleanup eval outputs (post-upload)
        if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}
        run: |
          if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
          rm -f meta_env.json || true
          # Remove any eval results JSONs that were moved into workspace
          rm -f results*.json || true
          rm -f -- ./*_report.json || true
          rm -f -- ./*_results.jsonl || true
          rm -f sample*.jsonl || true
          rm -f -- ./*_artifacts.tar.gz || true
          
      - name: Repair root-owned artifacts (post-run)
        if: ${{ always() && inputs.runner == 'cluster:mi355x-amds' }}
        run: |
          if [[ -f inferencex-e2e/configs/runners.yaml ]]; then cd inferencex-e2e; fi
          if [[ -d "$GITHUB_WORKSPACE" ]]; then
            sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE"
          fi

      - name: Slurm cleanup (post-run)
        if: always()
        run: *slurm-cleanup

      - name: Launcher cleanup (post-run)
        if: ${{ always() && env.INFERENCEX_LAUNCH_PYTHON != '' }}
        env:
          PYTHONPATH: ${{ github.workspace }}/.result-tooling/inferencex-e2e
        run: *launcher-cleanup