_run-ci.yml

name: CI Job

on:
  workflow_call:
    inputs:
      execute_command:
        type: string
        required: true
      runs_on:
        type: string
        required: false
        description: 'JSON-encoded list of runner labels (all required). e.g. ''["h100", "8gpu"]'' picks runners that carry BOTH labels. No default: GPU jobs must pass this explicitly.'
      container_image:
        type: string
        required: false
        default: 'radixark/miles:dev'
      skip_dependency_install:
        type: boolean
        required: false
        default: false
      ref:
        description: 'Git ref to check out (empty = the triggering commit). Set by release CI to test a release branch.'
        type: string
        required: false
        default: ''
      plan_already_resolved:
        description: 'Skip the --list-only preflight when a trusted caller already resolved the execution plan.'
        type: boolean
        required: false
        default: false
    secrets:
      WANDB_API_KEY:
        required: false
      NEON_DATABASE_URL:
        required: false

jobs:
  run:
    # fromJSON parses the input list so `runs-on` receives an actual array.
    # Without this, GitHub treats the string verbatim as one label literal
    # (e.g. the label name '["h100", "8gpu"]' with brackets), which no
    # runner carries, and the job queues forever.
    runs-on: ${{ fromJSON(inputs.runs_on) }}
    container:
      image: ${{ inputs.container_image }}
      options: >
        --gpus all
        --env CUDA_VISIBLE_DEVICES
        --ipc=host
        --shm-size=32g
        --ulimit memlock=-1
        --ulimit stack=67108864
        --memory=0
        --memory-swap=0
        -v /data/miles_ci:/data/miles_ci
        -v /data/miles_ci/models:/root/models
        -v /data/miles_ci/datasets:/root/datasets
        -v /data/miles_ci/hf_cache:/root/.cache/huggingface
        --privileged
        --ulimit nofile=65535:65535
    defaults:
      run:
        working-directory: ${{ github.workspace }}
    env:
      SGLANG_SOURCE_ROOT: /sgl-workspace/sglang/python
      MEGATRON_SOURCE_ROOT: /root/Megatron-LM
      GITHUB_COMMIT_NAME: ${{ inputs.ref || github.sha }}_${{ github.event.pull_request.number || github.event.inputs.pull_number || 'non-pr' }}
      WANDB_API_KEY: ${{ secrets.WANDB_API_KEY }}
      NEON_DATABASE_URL: ${{ secrets.NEON_DATABASE_URL }}
      MILES_TEST_ENABLE_INFINITE_RUN: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.infinite_run) || 'false' }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}

      - name: Configure source paths
        shell: bash
        run: echo "PYTHONPATH=$GITHUB_WORKSPACE:$SGLANG_SOURCE_ROOT:$MEGATRON_SOURCE_ROOT" >> "$GITHUB_ENV"

      - name: Cleanup Ray processes
        shell: bash
        run: |
          pkill -9 -f 'ray::' 2>/dev/null || true
          pkill -9 -f raylet 2>/dev/null || true
          pkill -9 -f gcs_server 2>/dev/null || true
          pkill -9 -f 'ray-dashboard' 2>/dev/null || true
          pkill -9 sglang 2>/dev/null || true
          ray stop --force 2>/dev/null || true
          sleep 3

      - name: Wait for GPU ready
        shell: bash
        run: |
          # When a self-hosted runner accepts a new job seconds after the
          # previous job's container exits, the NVIDIA driver subsystem may
          # still be tearing down its prior CUDA context. The next process
          # to call cudaGetDeviceCount() then hits Error 802 ("system not
          # yet initialized"). Block here until nvidia-smi succeeds, with
          # a bounded retry budget so a genuinely-broken GPU fails fast.
          for i in $(seq 1 60); do
            if nvidia-smi -L >/dev/null 2>&1; then
              echo "GPU ready after $((i-1)) retry(s)"
              nvidia-smi --query-gpu=index,name,memory.used --format=csv,noheader
              exit 0
            fi
            sleep 2
          done
          echo "ERROR: nvidia-smi did not become ready after 120s" >&2
          nvidia-smi 2>&1 | tail -20 || true
          exit 1

      - name: Resolve dependency refs
        if: ${{ !inputs.skip_dependency_install }}
        id: resolve-refs
        shell: bash
        env:
          PR_BODY: ${{ github.event.pull_request.body || '' }}
          INPUT_MEGATRON_PR: ${{ github.event.inputs.ci_megatron_pr || '' }}
          INPUT_SGLANG_PR: ${{ github.event.inputs.ci_sglang_pr || '' }}
        run: |
          MEGATRON_PR="${INPUT_MEGATRON_PR}"
          SGLANG_PR="${INPUT_SGLANG_PR}"
          if [ -n "$PR_BODY" ]; then
            PR_MEGATRON_PR=$(echo "$PR_BODY" | grep -m1 -oP '^ci-megatron-pr:\s+\K\S+' || true)
            PR_SGLANG_PR=$(echo "$PR_BODY" | grep -m1 -oP '^ci-sglang-pr:\s+\K\S+' || true)
            [ -z "$MEGATRON_PR" ] && [ -n "$PR_MEGATRON_PR" ] && MEGATRON_PR="$PR_MEGATRON_PR"
            [ -z "$SGLANG_PR" ] && [ -n "$PR_SGLANG_PR" ] && SGLANG_PR="$PR_SGLANG_PR"
          fi
          # Release branches carry release-lock.json (written at branch cut) pinning
          # the dependency SHAs the release was frozen with. Explicit overrides above
          # still win; the moving branch heads are only the last resort.
          if [ -f release-lock.json ]; then
            LOCK_SGLANG=$(python3 -c "import json; print(json.load(open('release-lock.json'))['sglang_commit'])")
            LOCK_MEGATRON=$(python3 -c "import json; print(json.load(open('release-lock.json'))['megatron_commit'])")
            [ -z "$SGLANG_PR" ] && SGLANG_PR="$LOCK_SGLANG"
            [ -z "$MEGATRON_PR" ] && MEGATRON_PR="$LOCK_MEGATRON"
            echo "release-lock.json present: sglang=$LOCK_SGLANG megatron=$LOCK_MEGATRON"
          fi
          [ -z "$MEGATRON_PR" ] && MEGATRON_PR="miles-main"
          [ -z "$SGLANG_PR" ] && SGLANG_PR="sglang-miles"
          resolve_fetch_ref() {
            local ref="$1"
            if [[ "$ref" =~ ^#([0-9]+)$ ]]; then
              echo "refs/pull/${BASH_REMATCH[1]}/head"
            else
              echo "$ref"
            fi
          }
          MEGATRON_FETCH=$(resolve_fetch_ref "$MEGATRON_PR")
          SGLANG_FETCH=$(resolve_fetch_ref "$SGLANG_PR")
          echo "ci_megatron_pr=$MEGATRON_FETCH" >> $GITHUB_OUTPUT
          echo "ci_sglang_pr=$SGLANG_FETCH" >> $GITHUB_OUTPUT
          echo "Resolved: megatron=$MEGATRON_PR -> fetch=$MEGATRON_FETCH, sglang=$SGLANG_PR -> fetch=$SGLANG_FETCH"

      - name: Reconcile Miles dependencies
        shell: bash
        run: |
          # The image pins cuDNN deliberately: transformer_engine's fused attention needs a
          # newer build than torch's own exact pin, so the Dockerfile installs it last. This
          # resolve drags it back down to whatever torch asks for, which makes fused attention
          # backward fail with CUDNN_STATUS_BAD_PARAM. Record what the image shipped and put it
          # back afterwards; --no-deps so nothing else in the resolved set moves.
          pinned=""
          for pkg in nvidia-cudnn-cu13 nvidia-cudnn-cu12; do
            v=$(python -m pip show "$pkg" 2>/dev/null | awk '/^Version:/{print $2}' || true)
            if [ -n "$v" ]; then pinned="$pinned $pkg==$v"; fi
          done
          python -m pip install -r "$GITHUB_WORKSPACE/requirements.txt" --break-system-packages
          for spec in $pinned; do
            pkg=${spec%%==*}
            want=${spec##*==}
            got=$(python -m pip show "$pkg" 2>/dev/null | awk '/^Version:/{print $2}' || true)
            if [ -n "$got" ] && [ "$got" != "$want" ]; then
              echo "Restoring image pin: $pkg $got -> $want"
              python -m pip install --no-deps --break-system-packages "$spec"
            fi
          done

      - name: Sync dependency sources
        if: ${{ !inputs.skip_dependency_install }}
        shell: bash
        env:
          MEGATRON_PR: ${{ steps.resolve-refs.outputs.ci_megatron_pr }}
          SGLANG_PR: ${{ steps.resolve-refs.outputs.ci_sglang_pr }}
        run: |
          cd /sgl-workspace/sglang && git reset --hard HEAD && git clean -fd && git fetch origin "$SGLANG_PR" && git checkout -f FETCH_HEAD && git log --oneline -1
          cd /root/Megatron-LM && git reset --hard HEAD && git clean -fd && git fetch origin "$MEGATRON_PR" && git checkout -f FETCH_HEAD && git log --oneline -1

      - name: Verify source resolution
        shell: bash
        run: python tests/ci/verify_source_resolution.py

      - name: Resolve suite plan
        if: ${{ !inputs.plan_already_resolved }}
        shell: bash
        run: ${{ inputs.execute_command }} --list-only

      - name: Run tests
        shell: bash
        run: ${{ inputs.execute_command }}