Repository navigation
fix(ci): wipe and re-clone the workspace when checkout fails #324
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: OperatorX Sweep | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pool: | |
| description: GPU runner pool (one physical node per shard) | |
| type: choice | |
| default: h100-dgxc | |
| options: [h100-dgxc, h200-dgxc, b200-nscale, b300, gb200, gb300, mi300x, mi325x, mi355x] | |
| backends: | |
| description: Comma-separated backends (torch GEMM/attention, AMD aiter attention, vllm routed MoE) | |
| type: string | |
| default: torch | |
| testlists: | |
| description: Comma-separated testlist stems | |
| type: string | |
| default: gemm | |
| world_sizes: | |
| description: Comma-separated world sizes (1,2,4; also 8 on eight-GPU pools) | |
| type: string | |
| default: '1' | |
| chunk_size: | |
| description: Maximum shapes per shard (1-500) | |
| type: number | |
| default: 500 | |
| recovery_run_id: | |
| description: Optional recent OperatorX run on this pool whose failed cleanup should be retried | |
| type: string | |
| default: '' | |
| pull_request: | |
| paths: | |
| - operatorx/** | |
| - .github/workflows/operatorx-sweep.yml | |
| permissions: | |
| contents: read | |
| actions: read # Download artifacts from a selected prior run for cleanup recovery. | |
| concurrency: | |
| group: operatorx-${{ github.event_name }}-${{ github.run_id }} | |
| cancel-in-progress: false | |
| env: | |
| PYTHONPATH: ${{ github.workspace }}/inferencex-e2e | |
| jobs: | |
| setup: | |
| name: Plan OperatorX coverage | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| outputs: | |
| matrix: ${{ steps.plan.outputs.matrix }} | |
| priority: ${{ steps.plan.outputs.priority }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| - name: Plan requested coverage | |
| id: plan | |
| env: | |
| OPERATORX_POOL: ${{ inputs.pool || 'h100-dgxc' }} | |
| OPERATORX_BACKENDS: ${{ inputs.backends || 'torch' }} | |
| OPERATORX_TESTLISTS: ${{ inputs.testlists || 'gemm' }} | |
| OPERATORX_WORLD_SIZES: ${{ inputs.world_sizes || '1' }} | |
| OPERATORX_RECOVERY_RUN: ${{ inputs.recovery_run_id }} | |
| OPERATORX_CHUNK_SIZE: ${{ github.event_name == 'workflow_dispatch' && format('{0}', inputs.chunk_size) || '500' }} | |
| PYTHONPATH: .:inferencex-e2e | |
| run: | | |
| set -eo pipefail | |
| source inferencex-e2e/benchmarks/check_env.sh | |
| check_env_vars OPERATORX_POOL OPERATORX_BACKENDS OPERATORX_TESTLISTS OPERATORX_WORLD_SIZES OPERATORX_CHUNK_SIZE GITHUB_RUN_ID GITHUB_RUN_ATTEMPT GITHUB_SHA GITHUB_OUTPUT | |
| python3 -m operatorx.ci plan \ | |
| --platform-config operatorx/platforms.json \ | |
| --pool "$OPERATORX_POOL" --backends "$OPERATORX_BACKENDS" \ | |
| --testlists "$OPERATORX_TESTLISTS" --world-sizes "$OPERATORX_WORLD_SIZES" \ | |
| --chunk-size "$OPERATORX_CHUNK_SIZE" --run-id "$GITHUB_RUN_ID" \ | |
| --attempt "$GITHUB_RUN_ATTEMPT" --source-sha "$GITHUB_SHA" \ | |
| --out operatorx-manifest.json > operatorx-matrix.json | |
| if [ -n "$OPERATORX_RECOVERY_RUN" ]; then | |
| [[ "$OPERATORX_RECOVERY_RUN" =~ ^[1-9][0-9]*$ ]] && [ "$(jq '.include | length' operatorx-matrix.json)" = 1 ] \ | |
| || { echo 'Recovery requires a numeric run ID and a single-shard selection' >&2; exit 1; } | |
| fi | |
| echo "matrix=$(cat operatorx-matrix.json)" >> "$GITHUB_OUTPUT" | |
| scored=$(printf '%s' '[{"runner":"operatorx","framework":"operatorx"}]' | | |
| uv run --no-project --exclude-newer PT12H --python 3.12 --with pyyaml python -m infx.workflows.ci_priority) | |
| echo "priority=$(jq -r '.[0].priority' <<<"$scored")" >> "$GITHUB_OUTPUT" | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: operatorx-manifest-${{ github.run_id }} | |
| path: operatorx-manifest.json | |
| overwrite: true | |
| if-no-files-found: error | |
| retention-days: 14 | |
| sweep: | |
| if: github.event_name == 'workflow_dispatch' | |
| needs: setup | |
| timeout-minutes: 70 | |
| strategy: | |
| fail-fast: false | |
| # Keep every shard eligible: a second matrix cap can strand priority labels | |
| # on GitHub-held jobs. Runner/node admission is owned by the scheduler. | |
| matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} | |
| runs-on: >- | |
| ${{ fromJSON(vars.PRIORITY_SCHEDULER_ENABLED == 'true' && | |
| format('["self-hosted",{0},"nodes:1",{1},{2}]', | |
| toJSON(matrix.pool), | |
| toJSON(format('ci-job-{0}-{1}', needs.setup.outputs.priority, matrix.queue-token)), | |
| toJSON(format('ci-attempt-{0}', github.run_attempt))) || | |
| format('["self-hosted",{0}]', toJSON(matrix.pool))) }} | |
| name: p${{ needs.setup.outputs.priority }} | OperatorX ${{ matrix.id }} | |
| env: | |
| OPERATORX_SHARD: ${{ matrix.id }} | |
| OPERATORX_OUTPUT: ${{ github.workspace }}/operatorx-output-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} | |
| PYTHONPATH: .:inferencex-e2e | |
| PYTHONDONTWRITEBYTECODE: '1' | |
| OPERATORX_CLEANUP_SECONDS: '180' | |
| steps: | |
| - name: Repair stale git state (pre-run) | |
| shell: bash | |
| run: | | |
| # Interrupted jobs leave stale locks and damaged git state in the | |
| # persistent workspace. actions/checkout only clears the top-level | |
| # index.lock/shallow.lock and recreates the repo when the top-level | |
| # clean/reset fails, yet every checkout also runs | |
| # `git submodule foreach --recursive` and walks every ref, so a | |
| # damaged submodule or a ref to an unreadable object fails the job. | |
| # One job runs per runner, so every lock here is stale. Keep | |
| # identical in benchmark-tmpl.yml, benchmark-multinode-tmpl.yml, | |
| # operatorx-sweep.yml, profile.yml and speedbench-al.yml. | |
| repo_dir="${GITHUB_WORKSPACE}" | |
| if [ -d "${repo_dir}/.git" ]; then | |
| git_in() { local dir="$1"; shift; git --git-dir="${dir}" --work-tree="${dir}" "$@"; } | |
| # Nested module git dirs are cleared when they are checked. | |
| clear_locks() { | |
| local lock n=0 | |
| while IFS= read -r -d '' lock; do | |
| rm -f "${lock}" && n=$((n + 1)) | |
| done < <(find "$1" -path "$1/modules" -prune -o -path '*/objects/[0-9a-f][0-9a-f]' -prune \ | |
| -o -name '*.lock' -type f -print0) | |
| echo "${n}" | |
| } | |
| # Checkout fetches the exact SHA, so remote-tracking refs and tags | |
| # are disposable and always go. Any other ref goes when rev-list, | |
| # which reads objects the way checkout's ref walk does, cannot | |
| # parse its object. Fails when refs cannot be listed or parsed. | |
| prune_refs() { | |
| local refs parsed prune | |
| refs=$(git_in "$1" for-each-ref --format='%(objectname) %(refname)') || return 1 | |
| parsed=$(awk '$2 !~ /^refs\/(remotes|tags)\// { print $1 }' <<<"${refs}" | | |
| git_in "$1" rev-list --no-walk --ignore-missing --stdin) || return 1 | |
| mapfile -t prune < <(awk 'NR == FNR { ok[$1] = 1; next } | |
| NF == 2 && ($2 ~ /^refs\/(remotes|tags)\// || !($1 in ok)) { print $2 }' \ | |
| <(printf '%s\n' "${parsed}") <(printf '%s\n' "${refs}")) | |
| if [ "${#prune[@]}" -gt 0 ]; then | |
| printf 'delete %s\n' "${prune[@]}" | git_in "$1" update-ref --no-deref --stdin || return 1 | |
| fi | |
| echo "${#prune[@]}" | |
| } | |
| # Prints one summary line per git dir and fails when it is damaged. | |
| # Every checkout reads a module's remote.origin.url, HEAD objects | |
| # and index; checkout recreates the repo for a bad top-level index. | |
| check_gitdir() { | |
| local locks pruned="" reason="" | |
| locks=$(clear_locks "$1") | |
| if [ "$2" = module ] && ! git_in "$1" config --local --get remote.origin.url >/dev/null 2>&1; then | |
| reason="no remote.origin.url" | |
| elif ! git_in "$1" rev-list --objects --no-walk HEAD >/dev/null 2>&1; then | |
| reason="HEAD objects unreadable" | |
| elif [ "$2" = module ] && ! git_in "$1" ls-files --stage >/dev/null 2>&1; then | |
| reason="index unreadable" | |
| elif ! pruned=$(prune_refs "$1" 2>/dev/null); then | |
| reason="refs unreadable" | |
| fi | |
| echo "[git-repair] ${1#"${repo_dir}/"}: ${locks} locks, ${pruned:-0} refs pruned, ${reason:+dropped: }${reason:-ok}" | |
| [ -z "${reason}" ] | |
| } | |
| # Module git dirs sit at any depth below .git/modules; names may | |
| # contain '/', so dirs without HEAD/config/objects/refs only group | |
| # them. | |
| is_gitdir() { [ -e "$1/HEAD" ] || [ -e "$1/config" ] || [ -e "$1/objects" ] || [ -e "$1/refs" ]; } | |
| drop_damaged_gitdirs() { | |
| local d wt root | |
| for d in "$1"/*/; do | |
| d="${d%/}" | |
| [ -d "${d}" ] && [ ! -L "${d}" ] || continue | |
| if ! is_gitdir "${d}"; then | |
| drop_damaged_gitdirs "${d}" | |
| elif check_gitdir "${d}" module; then | |
| drop_damaged_gitdirs "${d}/modules" | |
| else | |
| # Its worktree, even one no longer in .gitmodules, would keep | |
| # a .git file pointing at the removed git dir. | |
| if wt=$(git config -f "${d}/config" core.worktree 2>/dev/null) && | |
| wt=$(cd "${d}" && cd "${wt}" && pwd -P) && root=$(cd "${repo_dir}" && pwd -P) && | |
| [ -f "${wt}/.git" ] && [ "${wt#"${root}"/}" != "${wt}" ]; then | |
| echo "[git-repair] Removing its worktree: ${wt#"${root}"/}" | |
| rm -rf "${wt:?}" || true | |
| fi | |
| rm -rf "${d:?}" || true | |
| fi | |
| done | |
| } | |
| # `submodule update` re-clones only into an empty worktree, so a | |
| # worktree whose git dir is gone or partial, or whose .git file | |
| # does not resolve, is removed together with that git dir. | |
| drop_orphaned_worktrees() { | |
| local entry name path wt gd | |
| [ -f "$1/.gitmodules" ] || return 0 | |
| while IFS= read -r -d '' entry; do | |
| name="${entry%%$'\n'*}" | |
| name="${name#submodule.}" | |
| name="${name%.path}" | |
| path="${entry#*$'\n'}" | |
| case "/${name}/ /${path}/" in *//*|*/./*|*/../*) continue ;; esac | |
| wt="$1/${path}" | |
| gd="$2/modules/${name}" | |
| if is_gitdir "${gd}" && | |
| { [ ! -f "${wt}/.git" ] || git -C "${wt}" rev-parse --git-dir >/dev/null 2>&1; }; then | |
| drop_orphaned_worktrees "${wt}" "${gd}" | |
| elif [ -e "${gd}" ] || [ -n "$(ls -A "${wt}" 2>/dev/null)" ]; then | |
| echo "[git-repair] Removing damaged submodule: ${wt#"${repo_dir}/"}" | |
| rm -rf "${gd:?}" "${wt:?}" || true | |
| fi | |
| done < <(git config -z -f "$1/.gitmodules" --get-regexp '^submodule\..*\.path$' 2>/dev/null) | |
| } | |
| # A damaged top-level repo is removed, so checkout re-clones it | |
| # instead of failing. | |
| if check_gitdir "${repo_dir}/.git" top-level; then | |
| drop_damaged_gitdirs "${repo_dir:?}/.git/modules" | |
| drop_orphaned_worktrees "${repo_dir:?}" "${repo_dir:?}/.git" | |
| else | |
| rm -rf "${repo_dir:?}/.git" || true | |
| fi | |
| fi | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.sha }} | |
| persist-credentials: false | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: operatorx-manifest-${{ github.run_id }} | |
| path: operatorx-control | |
| - name: Download prior execution for recovery | |
| if: inputs.recovery_run_id != '' | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| github-token: ${{ github.token }} | |
| run-id: ${{ inputs.recovery_run_id }} | |
| pattern: operatorx-shard-${{ inputs.recovery_run_id }}-* | |
| path: operatorx-recovery | |
| - name: Retry cleanup of the selected prior run | |
| if: inputs.recovery_run_id != '' | |
| env: | |
| OPERATORX_RECOVERY_RUN: ${{ inputs.recovery_run_id }} | |
| OPERATORX_POOL: ${{ matrix.pool }} | |
| run: | | |
| set -eo pipefail | |
| source inferencex-e2e/benchmarks/check_env.sh | |
| check_env_vars RUNNER_TEMP OPERATORX_RECOVERY_RUN OPERATORX_POOL OPERATORX_CLEANUP_SECONDS | |
| python3 -m operatorx.ci recover --artifacts operatorx-recovery \ | |
| --run-id "$OPERATORX_RECOVERY_RUN" --pool "$OPERATORX_POOL" \ | |
| --cleanup-seconds "$OPERATORX_CLEANUP_SECONDS" \ | |
| --platform-config operatorx/platforms.json | |
| - name: Run one Slurm allocation | |
| run: | | |
| set -eo pipefail | |
| source inferencex-e2e/benchmarks/check_env.sh | |
| check_env_vars RUNNER_TEMP OPERATORX_SHARD OPERATORX_OUTPUT OPERATORX_CLEANUP_SECONDS GITHUB_RUN_ID GITHUB_RUN_ATTEMPT GITHUB_SHA RUNNER_NAME | |
| python3 -m operatorx.ci execute \ | |
| --platform-config operatorx/platforms.json \ | |
| --manifest operatorx-control/operatorx-manifest.json --shard "$OPERATORX_SHARD" \ | |
| --output "$OPERATORX_OUTPUT" --run-id "$GITHUB_RUN_ID" \ | |
| --attempt "$GITHUB_RUN_ATTEMPT" --source-sha "$GITHUB_SHA" \ | |
| --runner-name "$RUNNER_NAME" --time-minutes 45 \ | |
| --cleanup-seconds "$OPERATORX_CLEANUP_SECONDS" | |
| - name: Recover allocation and partial results | |
| if: always() | |
| run: | | |
| set -eo pipefail | |
| source inferencex-e2e/benchmarks/check_env.sh | |
| check_env_vars OPERATORX_OUTPUT OPERATORX_CLEANUP_SECONDS | |
| python3 -m operatorx.ci finalize --output "$OPERATORX_OUTPUT" \ | |
| --cleanup-seconds "$OPERATORX_CLEANUP_SECONDS" | |
| - name: Capture scheduler diagnostics on failure | |
| if: failure() | |
| run: | | |
| set -eo pipefail | |
| source inferencex-e2e/benchmarks/check_env.sh | |
| check_env_vars OPERATORX_OUTPUT | |
| if [ -d "$OPERATORX_OUTPUT" ]; then | |
| timeout 15 sinfo --Node --noheader --format='%N|%P|%t' \ | |
| > "$OPERATORX_OUTPUT/slurm-nodes.log" 2>&1 || true | |
| fi | |
| - name: Upload results and diagnostics | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: operatorx-shard-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} | |
| path: ${{ env.OPERATORX_OUTPUT }} | |
| if-no-files-found: error | |
| retention-days: 14 | |
| summarize: | |
| name: Summarize OperatorX coverage | |
| if: always() && github.event_name == 'workflow_dispatch' && needs.setup.result == 'success' | |
| needs: [setup, sweep] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.sha }} | |
| persist-credentials: false | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: operatorx-manifest-${{ github.run_id }} | |
| path: operatorx-control | |
| - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: operatorx-shard-${{ github.run_id }}-* | |
| path: operatorx-artifacts | |
| - name: Report latest attempt for every requested shard | |
| env: | |
| PYTHONPATH: .:inferencex-e2e | |
| run: | | |
| set -eo pipefail | |
| source inferencex-e2e/benchmarks/check_env.sh | |
| check_env_vars GITHUB_STEP_SUMMARY | |
| python3 -m operatorx.ci summarize \ | |
| --manifest operatorx-control/operatorx-manifest.json \ | |
| --artifacts operatorx-artifacts --out operatorx-summary.json >> "$GITHUB_STEP_SUMMARY" | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: always() | |
| with: | |
| name: operatorx-summary-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: operatorx-summary.json | |
| if-no-files-found: error | |
| retention-days: 14 |