benchmark-tmpl.yml

name: Template - Benchmark
on:
  workflow_call:
    inputs:
      runner:
        required: true
        type: string
      image:
        required: true
        type: string
      model:
        required: true
        type: string
      model-prefix:
        required: true
        type: string
      precision:
        required: true
        type: string
      framework:
        required: true
        type: string
      exp-name:
        required: true
        type: string
      isl:
        required: true
        type: string
      osl:
        required: true
        type: string
      tp:
        required: true
        type: string
      ep:
        required: true
        type: string
      dp-attn:
        required: true
        type: boolean
      max-model-len:
        required: true
        type: string
      conc:
        required: true
        type: string
      spec-decoding:
        required: true
        type: string
      disagg:
        required: true
        type: string
      run-eval:
        type: boolean
        required: true
        default: false
      eval-only:
        description: "Run only evals (skip throughput benchmark)"
        type: boolean
        required: false
        default: false
      random-range-ratio:
        required: false
        type: string
        default: '0.8'
      ref:
        description: "Git ref (branch/sha) to checkout"
        required: false
        type: string
      scenario-type:
        description: "Scenario type (fixed-seq-len or agentic-coding)"
        required: false
        type: string
        default: 'fixed-seq-len'
      offloading:
        description: "KV offload backend for agentic scenarios (none/cpu/ssd)"
        required: false
        type: string
        default: 'none'
      total-cpu-dram-gb:
        description: "Total CPU DRAM in GB for KV offloading"
        required: false
        type: string
        default: '600'
      duration:
        description: "Benchmark duration in seconds"
        required: false
        type: string
        default: '1800'
env:
  RANDOM_RANGE_RATIO: 0.8
  HF_TOKEN: ${{ secrets.HF_TOKEN }}
  HF_HUB_CACHE: '/mnt/hf_hub_cache/'
  EXP_NAME: ${{ inputs.exp-name }}
  MODEL: ${{ inputs.model }}
  MODEL_PREFIX: ${{ inputs.model-prefix }}
  ISL: ${{ inputs.isl }}
  OSL: ${{ inputs.osl }}
  MAX_MODEL_LEN: ${{ inputs.max-model-len }}
  IMAGE: ${{ inputs.image }}
  FRAMEWORK: ${{ inputs.framework }}
  PRECISION: ${{ inputs.precision }}
  TP: ${{ inputs.tp }}
  EP_SIZE: ${{ inputs.ep }}
  DP_ATTENTION: ${{ inputs.dp-attn }}
  CONC: ${{ inputs.conc }}
  SPEC_DECODING: ${{ inputs.spec-decoding }}
  DISAGG: ${{ inputs.disagg }}
  RUN_EVAL: ${{ inputs.run-eval }}
  EVAL_ONLY: ${{ inputs.eval-only }}
  SCENARIO_TYPE: ${{ inputs.scenario-type }}
  SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || '' }}
  IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }}
  OFFLOADING: ${{ inputs.offloading }}
  TOTAL_CPU_DRAM_GB: ${{ inputs.total-cpu-dram-gb }}
  DURATION: ${{ inputs.duration }}
  RESULT_DIR: /workspace/results
  PYTHONDONTWRITEBYTECODE: '1'
  PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache

permissions:
  contents: read

jobs:
  benchmark:
    runs-on: ${{ inputs.runner }}
    timeout-minutes: 500
    name: "${{ inputs.exp-name }} ${{ inputs.precision }} ${{ inputs.runner }} ${{ inputs.framework }} | tp=${{ inputs.tp }} ep=${{ inputs.ep }} dpa=${{ inputs.dp-attn }} | disagg-${{ inputs.disagg }} spec-${{ inputs.spec-decoding }} conc-${{ inputs.conc }}${{ inputs.eval-only && ' | eval-only' || (inputs.run-eval && ' | eval' || '') }}"
    steps:
      - name: Resource cleanup (pre-run)
        run: &resource-cleanup |
          # Cleanup Docker resources
          if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
            echo "[Docker] Cleaning up resources ..."
            docker ps -aq | xargs -r docker rm -f
            docker network prune -f
            while [ -n "$(docker ps -aq)" ]; do
              docker ps -a
              sleep 5
            done
          fi

          # Cleanup SLURM resources
          if command -v squeue >/dev/null 2>&1; then
            echo "[Slurm] Cleaning up jobs with name: ${{ runner.name }} ..."
            scancel --name="${{ runner.name }}" || true
            while [ -n "$(squeue --name='${{ runner.name }}' --noheader --format='%i')" ]; do
              squeue --name="${{ runner.name }}"
              sleep 5
            done
          fi

          # Cleanup results/ from a prior job on this runner. Agentic jobs
          # write to fixed subpaths (trace_replay/, metrics_*, etc.), so stale
          # data from a previous job would otherwise be picked up as this
          # job's output when replay fails early.
          rm -rf "${{ github.workspace }}/results" 2>/dev/null || true

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          token: ${{ secrets.REPO_PAT }}
          fetch-depth: 0
          ref: ${{ inputs.ref || github.sha }}
          clean: true
          submodules: true

      - name: Cleanup stale outputs (pre-run)
        run: |
          rm -f meta_env.json || true
          rm -f results*.json || true
          rm -f sample*.jsonl || true
          rm -f server.log || true
          rm -f gpu_metrics.csv || true

      - name: Launch job script
        env:
          RUNNER_NAME: ${{ runner.name }}
          RUNNER_TYPE: ${{ inputs.runner }}
          # Hash uniquely on {EXP_NAME}_{PRECISION}_{FRAMEWORK}_tp{}-ep{}-dpa{}_disagg-{}_spec-{}_conc{}_{runner}
          RESULT_FILENAME: ${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_tp${{ env.TP }}-ep${{ env.EP_SIZE }}-dpa${{ env.DP_ATTENTION }}_disagg-${{ env.DISAGG }}_spec-${{ env.SPEC_DECODING }}_conc${{ env.CONC }}_${{ runner.name }}
          # Suppress per-job eval markdown from being appended to the step summary.
          # We'll publish a single combined eval table in the collection job instead.
          GITHUB_STEP_SUMMARY: ''
        run: |
          # Export RESULT_FILENAME early so it's available for artifact uploads even if cancelled
          echo "RESULT_FILENAME=${RESULT_FILENAME}" >> $GITHUB_ENV

          bash ./runners/launch_${RUNNER_NAME%%_*}.sh

          if [ "${{ inputs.eval-only }}" = "true" ]; then
            echo "Eval-only mode: skipping benchmark result file check"
            # Verify eval produced results
            if ! ls results*.json 1>/dev/null 2>&1; then
              echo "Eval-only run failed: no results*.json files found." >&2
              exit 1
            fi
          else
            FOUND_RESULT_FILE=
            for i in {1..10}; do
              if [ -f "$RESULT_FILENAME.json" ]; then
                FOUND_RESULT_FILE=true
                break
              fi
              echo "Waiting for result file... (attempt $i)"
              sleep 1
            done

            if [ -z "$FOUND_RESULT_FILE" ]; then
              echo "Run failed: Benchmark result $RESULT_FILENAME.json not found." >&2
              exit 1
            fi
          fi

      - name: Process result
        if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }}
        env:
          RUNNER_TYPE: ${{ inputs.runner }}
        run: |
          python3 utils/process_result.py

      - name: Upload result
        if: ${{ !inputs.eval-only && inputs.scenario-type != 'agentic-coding' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bmk_${{ env.RESULT_FILENAME }}
          path: agg_${{ env.RESULT_FILENAME }}.json

      - name: Upload agentic aggregated result
        if: ${{ inputs.scenario-type == 'agentic-coding' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: bmk_agentic_${{ env.RESULT_FILENAME }}
          path: ${{ env.RESULT_FILENAME }}.json

      - name: Upload agentic raw results
        if: ${{ always() && inputs.scenario-type == 'agentic-coding' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: agentic_${{ env.RESULT_FILENAME }}
          path: |
            results/server.log
            results/metrics_server_metrics.csv
            results/metrics_plots.png
            results/metrics_workload.png
            results/metrics_client_metrics.csv
            results/benchmark.log
            results/config.yaml
            results/vllm_command.txt
            results/benchmark_command.txt
            results/workload_distribution_summary.txt
            results/workload_distribution_plots.png
            results/trace_replay/detailed_results.csv
            results/trace_replay/debug_trace.jsonl
          if-no-files-found: ignore

      - name: Upload server logs
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ inputs.eval-only && 'eval_server_logs_' || 'server_logs_' }}${{ env.RESULT_FILENAME }}
          path: ${{ inputs.scenario-type == 'agentic-coding' && 'results/server.log' || 'server.log' }}
          if-no-files-found: ignore

      - name: Upload GPU metrics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }}
          path: gpu_metrics.csv
          if-no-files-found: ignore

      - name: Upload eval results (if any)
        if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: eval_${{ env.EXP_NAME }}_${{ env.RESULT_FILENAME }}
          path: |
            meta_env.json
            results*.json
            sample*.jsonl
          if-no-files-found: ${{ inputs.eval-only && 'error' || 'ignore' }}

      - name: Verify eval scores
        if: ${{ (success() || failure()) && inputs.eval-only }}
        run: python3 utils/evals/validate_scores.py

      - name: Cleanup eval outputs (post-upload)
        if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}
        run: |
          rm -f meta_env.json || true
          # Remove any eval results JSONs that were moved into workspace
          rm -f results*.json || true
          rm -f sample*.jsonl || true
          
      - name: Resource cleanup (post-run)
        if: always()
        run: *resource-cleanup