Skip to content

Commit e123d80

Browse files
adibarraadibarra
committed
fix(ci): repair corrupt submodule state before checkout
gb200-nv_17 failed every job at actions/checkout because .git/modules/utils/aiperf/index was truncated. HEAD still resolved, so the existing pre-run repair kept the module, and actions/checkout then died in `git submodule foreach --recursive`, which removeAuth() runs on every checkout before any recovery path. The pre-run step now removes every stale *.lock under .git at any depth, drops module git dirs at any depth whose remote.origin.url, HEAD objects or index cannot be read, and removes submodule worktrees whose git dir is missing or partial or whose .git file does not resolve, so checkout re-clones only the damaged submodule. The top-level repository stays with actions/checkout, which already recreates it when its index, HEAD or config is damaged. Every self-hosted job that runs actions/checkout in the shared workspace now carries the identical step: both benchmark templates, speedbench-al, profile and operatorx-sweep.
1 parent ddaccac commit e123d80

5 files changed

Lines changed: 337 additions & 38 deletions

File tree

‎.github/workflows/benchmark-multinode-tmpl.yml‎

Lines changed: 64 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -231,28 +231,73 @@ jobs:
231231
fi
232232

233233
- name: Repair stale git state (pre-run)
234+
shell: bash
234235
run: |
235-
# Interrupted runs leave index.lock files and corrupt submodule git
236-
# dirs behind in the persistent self-hosted workspace, and both make
237-
# actions/checkout fail before any benchmark starts. The runner
238-
# executes one job at a time, so any lock found here is stale by
239-
# definition. Grouping dirs (e.g. .git/modules/utils) hold only
240-
# nested module dirs, so any dir carrying top-level files is a
241-
# module git dir — a truncated clone may lack HEAD entirely, and a
242-
# git dir whose HEAD does not resolve to a commit is dropped
243-
# together with its worktree so checkout re-clones it.
236+
# Interrupted jobs leave stale locks and damaged submodule git state
237+
# in the persistent workspace. actions/checkout only clears the
238+
# top-level index.lock/shallow.lock and recreates the repo when the
239+
# top-level clean/reset fails, yet every checkout also runs
240+
# `git submodule foreach --recursive`, which dies on any damaged
241+
# submodule. One job runs per runner, so every lock here is stale.
242+
# Keep identical in benchmark-tmpl.yml, benchmark-multinode-tmpl.yml,
243+
# operatorx-sweep.yml, profile.yml and speedbench-al.yml.
244244
repo_dir="${GITHUB_WORKSPACE}"
245245
if [ -d "${repo_dir}/.git" ]; then
246-
find "${repo_dir}/.git" -name index.lock -type f -delete || true
247-
for gitdir in "${repo_dir}"/.git/modules/* "${repo_dir}"/.git/modules/*/*; do
248-
[ -d "${gitdir}" ] || continue
249-
[ -n "$(find "${gitdir}" -maxdepth 1 -type f -print -quit)" ] || continue
250-
if ! git --git-dir="${gitdir}" rev-parse --verify 'HEAD^{commit}' >/dev/null 2>&1; then
251-
rel="${gitdir##*/.git/modules/}"
252-
echo "[git-repair] Removing corrupt submodule state: ${rel}"
253-
rm -rf "${gitdir}" "${repo_dir:?}/${rel}" "${repo_dir:?}/inferencex-e2e/${rel}"
254-
fi
255-
done
246+
while IFS= read -r -d '' lock; do
247+
echo "[git-repair] Removing stale lock: ${lock#"${repo_dir}/"}"
248+
rm -f "${lock}" || true
249+
done < <(find "${repo_dir}/.git" -path '*/objects/[0-9a-f][0-9a-f]' -prune \
250+
-o -name '*.lock' -type f -print0)
251+
# Module git dirs sit at any depth below .git/modules; names may
252+
# contain '/', so dirs without HEAD/config/objects/refs only group
253+
# them. Every checkout reads each module's remote.origin.url, HEAD
254+
# objects and index, so a git dir missing any of them is damaged.
255+
is_gitdir() { [ -e "$1/HEAD" ] || [ -e "$1/config" ] || [ -e "$1/objects" ] || [ -e "$1/refs" ]; }
256+
gitdir_ok() {
257+
git --git-dir="$1" --work-tree="$1" config --local --get remote.origin.url >/dev/null 2>&1 &&
258+
git --git-dir="$1" --work-tree="$1" rev-list --objects --no-walk HEAD >/dev/null 2>&1 &&
259+
git --git-dir="$1" --work-tree="$1" ls-files --stage >/dev/null 2>&1
260+
}
261+
drop_damaged_gitdirs() {
262+
local d
263+
for d in "$1"/*/; do
264+
d="${d%/}"
265+
[ -d "${d}" ] && [ ! -L "${d}" ] || continue
266+
if ! is_gitdir "${d}"; then
267+
drop_damaged_gitdirs "${d}"
268+
elif gitdir_ok "${d}"; then
269+
drop_damaged_gitdirs "${d}/modules"
270+
else
271+
echo "[git-repair] Removing damaged git dir: ${d#"${repo_dir}/"}"
272+
rm -rf "${d:?}" || true
273+
fi
274+
done
275+
}
276+
# `submodule update` re-clones only into an empty worktree, so a
277+
# worktree whose git dir is gone or partial, or whose .git file
278+
# does not resolve, is removed together with that git dir.
279+
drop_orphaned_worktrees() {
280+
local entry name path wt gd
281+
[ -f "$1/.gitmodules" ] || return 0
282+
while IFS= read -r -d '' entry; do
283+
name="${entry%%$'\n'*}"
284+
name="${name#submodule.}"
285+
name="${name%.path}"
286+
path="${entry#*$'\n'}"
287+
case "/${name}/ /${path}/" in *//*|*/./*|*/../*) continue ;; esac
288+
wt="$1/${path}"
289+
gd="$2/modules/${name}"
290+
if is_gitdir "${gd}" &&
291+
{ [ ! -f "${wt}/.git" ] || git -C "${wt}" rev-parse --git-dir >/dev/null 2>&1; }; then
292+
drop_orphaned_worktrees "${wt}" "${gd}"
293+
elif [ -e "${gd}" ] || [ -n "$(ls -A "${wt}" 2>/dev/null)" ]; then
294+
echo "[git-repair] Removing damaged submodule: ${wt#"${repo_dir}/"}"
295+
rm -rf "${gd:?}" "${wt:?}" || true
296+
fi
297+
done < <(git config -z -f "$1/.gitmodules" --get-regexp '^submodule\..*\.path$' 2>/dev/null)
298+
}
299+
drop_damaged_gitdirs "${repo_dir:?}/.git/modules"
300+
drop_orphaned_worktrees "${repo_dir:?}" "${repo_dir:?}/.git"
256301
fi
257302
258303
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

‎.github/workflows/benchmark-tmpl.yml‎

Lines changed: 64 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -188,28 +188,73 @@ jobs:
188188
fi
189189

190190
- name: Repair stale git state (pre-run)
191+
shell: bash
191192
run: |
192-
# Interrupted runs leave index.lock files and corrupt submodule git
193-
# dirs behind in the persistent self-hosted workspace, and both make
194-
# actions/checkout fail before any benchmark starts. The runner
195-
# executes one job at a time, so any lock found here is stale by
196-
# definition. Grouping dirs (e.g. .git/modules/utils) hold only
197-
# nested module dirs, so any dir carrying top-level files is a
198-
# module git dir — a truncated clone may lack HEAD entirely, and a
199-
# git dir whose HEAD does not resolve to a commit is dropped
200-
# together with its worktree so checkout re-clones it.
193+
# Interrupted jobs leave stale locks and damaged submodule git state
194+
# in the persistent workspace. actions/checkout only clears the
195+
# top-level index.lock/shallow.lock and recreates the repo when the
196+
# top-level clean/reset fails, yet every checkout also runs
197+
# `git submodule foreach --recursive`, which dies on any damaged
198+
# submodule. One job runs per runner, so every lock here is stale.
199+
# Keep identical in benchmark-tmpl.yml, benchmark-multinode-tmpl.yml,
200+
# operatorx-sweep.yml, profile.yml and speedbench-al.yml.
201201
repo_dir="${GITHUB_WORKSPACE}"
202202
if [ -d "${repo_dir}/.git" ]; then
203-
find "${repo_dir}/.git" -name index.lock -type f -delete || true
204-
for gitdir in "${repo_dir}"/.git/modules/* "${repo_dir}"/.git/modules/*/*; do
205-
[ -d "${gitdir}" ] || continue
206-
[ -n "$(find "${gitdir}" -maxdepth 1 -type f -print -quit)" ] || continue
207-
if ! git --git-dir="${gitdir}" rev-parse --verify 'HEAD^{commit}' >/dev/null 2>&1; then
208-
rel="${gitdir##*/.git/modules/}"
209-
echo "[git-repair] Removing corrupt submodule state: ${rel}"
210-
rm -rf "${gitdir}" "${repo_dir:?}/${rel}" "${repo_dir:?}/inferencex-e2e/${rel}"
211-
fi
212-
done
203+
while IFS= read -r -d '' lock; do
204+
echo "[git-repair] Removing stale lock: ${lock#"${repo_dir}/"}"
205+
rm -f "${lock}" || true
206+
done < <(find "${repo_dir}/.git" -path '*/objects/[0-9a-f][0-9a-f]' -prune \
207+
-o -name '*.lock' -type f -print0)
208+
# Module git dirs sit at any depth below .git/modules; names may
209+
# contain '/', so dirs without HEAD/config/objects/refs only group
210+
# them. Every checkout reads each module's remote.origin.url, HEAD
211+
# objects and index, so a git dir missing any of them is damaged.
212+
is_gitdir() { [ -e "$1/HEAD" ] || [ -e "$1/config" ] || [ -e "$1/objects" ] || [ -e "$1/refs" ]; }
213+
gitdir_ok() {
214+
git --git-dir="$1" --work-tree="$1" config --local --get remote.origin.url >/dev/null 2>&1 &&
215+
git --git-dir="$1" --work-tree="$1" rev-list --objects --no-walk HEAD >/dev/null 2>&1 &&
216+
git --git-dir="$1" --work-tree="$1" ls-files --stage >/dev/null 2>&1
217+
}
218+
drop_damaged_gitdirs() {
219+
local d
220+
for d in "$1"/*/; do
221+
d="${d%/}"
222+
[ -d "${d}" ] && [ ! -L "${d}" ] || continue
223+
if ! is_gitdir "${d}"; then
224+
drop_damaged_gitdirs "${d}"
225+
elif gitdir_ok "${d}"; then
226+
drop_damaged_gitdirs "${d}/modules"
227+
else
228+
echo "[git-repair] Removing damaged git dir: ${d#"${repo_dir}/"}"
229+
rm -rf "${d:?}" || true
230+
fi
231+
done
232+
}
233+
# `submodule update` re-clones only into an empty worktree, so a
234+
# worktree whose git dir is gone or partial, or whose .git file
235+
# does not resolve, is removed together with that git dir.
236+
drop_orphaned_worktrees() {
237+
local entry name path wt gd
238+
[ -f "$1/.gitmodules" ] || return 0
239+
while IFS= read -r -d '' entry; do
240+
name="${entry%%$'\n'*}"
241+
name="${name#submodule.}"
242+
name="${name%.path}"
243+
path="${entry#*$'\n'}"
244+
case "/${name}/ /${path}/" in *//*|*/./*|*/../*) continue ;; esac
245+
wt="$1/${path}"
246+
gd="$2/modules/${name}"
247+
if is_gitdir "${gd}" &&
248+
{ [ ! -f "${wt}/.git" ] || git -C "${wt}" rev-parse --git-dir >/dev/null 2>&1; }; then
249+
drop_orphaned_worktrees "${wt}" "${gd}"
250+
elif [ -e "${gd}" ] || [ -n "$(ls -A "${wt}" 2>/dev/null)" ]; then
251+
echo "[git-repair] Removing damaged submodule: ${wt#"${repo_dir}/"}"
252+
rm -rf "${gd:?}" "${wt:?}" || true
253+
fi
254+
done < <(git config -z -f "$1/.gitmodules" --get-regexp '^submodule\..*\.path$' 2>/dev/null)
255+
}
256+
drop_damaged_gitdirs "${repo_dir:?}/.git/modules"
257+
drop_orphaned_worktrees "${repo_dir:?}" "${repo_dir:?}/.git"
213258
fi
214259
215260
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

‎.github/workflows/operatorx-sweep.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,75 @@ jobs:
118118
PYTHONDONTWRITEBYTECODE: '1'
119119
OPERATORX_CLEANUP_SECONDS: '180'
120120
steps:
121+
- name: Repair stale git state (pre-run)
122+
shell: bash
123+
run: |
124+
# Interrupted jobs leave stale locks and damaged submodule git state
125+
# in the persistent workspace. actions/checkout only clears the
126+
# top-level index.lock/shallow.lock and recreates the repo when the
127+
# top-level clean/reset fails, yet every checkout also runs
128+
# `git submodule foreach --recursive`, which dies on any damaged
129+
# submodule. One job runs per runner, so every lock here is stale.
130+
# Keep identical in benchmark-tmpl.yml, benchmark-multinode-tmpl.yml,
131+
# operatorx-sweep.yml, profile.yml and speedbench-al.yml.
132+
repo_dir="${GITHUB_WORKSPACE}"
133+
if [ -d "${repo_dir}/.git" ]; then
134+
while IFS= read -r -d '' lock; do
135+
echo "[git-repair] Removing stale lock: ${lock#"${repo_dir}/"}"
136+
rm -f "${lock}" || true
137+
done < <(find "${repo_dir}/.git" -path '*/objects/[0-9a-f][0-9a-f]' -prune \
138+
-o -name '*.lock' -type f -print0)
139+
# Module git dirs sit at any depth below .git/modules; names may
140+
# contain '/', so dirs without HEAD/config/objects/refs only group
141+
# them. Every checkout reads each module's remote.origin.url, HEAD
142+
# objects and index, so a git dir missing any of them is damaged.
143+
is_gitdir() { [ -e "$1/HEAD" ] || [ -e "$1/config" ] || [ -e "$1/objects" ] || [ -e "$1/refs" ]; }
144+
gitdir_ok() {
145+
git --git-dir="$1" --work-tree="$1" config --local --get remote.origin.url >/dev/null 2>&1 &&
146+
git --git-dir="$1" --work-tree="$1" rev-list --objects --no-walk HEAD >/dev/null 2>&1 &&
147+
git --git-dir="$1" --work-tree="$1" ls-files --stage >/dev/null 2>&1
148+
}
149+
drop_damaged_gitdirs() {
150+
local d
151+
for d in "$1"/*/; do
152+
d="${d%/}"
153+
[ -d "${d}" ] && [ ! -L "${d}" ] || continue
154+
if ! is_gitdir "${d}"; then
155+
drop_damaged_gitdirs "${d}"
156+
elif gitdir_ok "${d}"; then
157+
drop_damaged_gitdirs "${d}/modules"
158+
else
159+
echo "[git-repair] Removing damaged git dir: ${d#"${repo_dir}/"}"
160+
rm -rf "${d:?}" || true
161+
fi
162+
done
163+
}
164+
# `submodule update` re-clones only into an empty worktree, so a
165+
# worktree whose git dir is gone or partial, or whose .git file
166+
# does not resolve, is removed together with that git dir.
167+
drop_orphaned_worktrees() {
168+
local entry name path wt gd
169+
[ -f "$1/.gitmodules" ] || return 0
170+
while IFS= read -r -d '' entry; do
171+
name="${entry%%$'\n'*}"
172+
name="${name#submodule.}"
173+
name="${name%.path}"
174+
path="${entry#*$'\n'}"
175+
case "/${name}/ /${path}/" in *//*|*/./*|*/../*) continue ;; esac
176+
wt="$1/${path}"
177+
gd="$2/modules/${name}"
178+
if is_gitdir "${gd}" &&
179+
{ [ ! -f "${wt}/.git" ] || git -C "${wt}" rev-parse --git-dir >/dev/null 2>&1; }; then
180+
drop_orphaned_worktrees "${wt}" "${gd}"
181+
elif [ -e "${gd}" ] || [ -n "$(ls -A "${wt}" 2>/dev/null)" ]; then
182+
echo "[git-repair] Removing damaged submodule: ${wt#"${repo_dir}/"}"
183+
rm -rf "${gd:?}" "${wt:?}" || true
184+
fi
185+
done < <(git config -z -f "$1/.gitmodules" --get-regexp '^submodule\..*\.path$' 2>/dev/null)
186+
}
187+
drop_damaged_gitdirs "${repo_dir:?}/.git/modules"
188+
drop_orphaned_worktrees "${repo_dir:?}" "${repo_dir:?}/.git"
189+
fi
121190
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
122191
with:
123192
ref: ${{ github.sha }}

‎.github/workflows/profile.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,76 @@ jobs:
243243
done
244244
fi
245245
246+
- name: Repair stale git state (pre-run)
247+
shell: bash
248+
run: |
249+
# Interrupted jobs leave stale locks and damaged submodule git state
250+
# in the persistent workspace. actions/checkout only clears the
251+
# top-level index.lock/shallow.lock and recreates the repo when the
252+
# top-level clean/reset fails, yet every checkout also runs
253+
# `git submodule foreach --recursive`, which dies on any damaged
254+
# submodule. One job runs per runner, so every lock here is stale.
255+
# Keep identical in benchmark-tmpl.yml, benchmark-multinode-tmpl.yml,
256+
# operatorx-sweep.yml, profile.yml and speedbench-al.yml.
257+
repo_dir="${GITHUB_WORKSPACE}"
258+
if [ -d "${repo_dir}/.git" ]; then
259+
while IFS= read -r -d '' lock; do
260+
echo "[git-repair] Removing stale lock: ${lock#"${repo_dir}/"}"
261+
rm -f "${lock}" || true
262+
done < <(find "${repo_dir}/.git" -path '*/objects/[0-9a-f][0-9a-f]' -prune \
263+
-o -name '*.lock' -type f -print0)
264+
# Module git dirs sit at any depth below .git/modules; names may
265+
# contain '/', so dirs without HEAD/config/objects/refs only group
266+
# them. Every checkout reads each module's remote.origin.url, HEAD
267+
# objects and index, so a git dir missing any of them is damaged.
268+
is_gitdir() { [ -e "$1/HEAD" ] || [ -e "$1/config" ] || [ -e "$1/objects" ] || [ -e "$1/refs" ]; }
269+
gitdir_ok() {
270+
git --git-dir="$1" --work-tree="$1" config --local --get remote.origin.url >/dev/null 2>&1 &&
271+
git --git-dir="$1" --work-tree="$1" rev-list --objects --no-walk HEAD >/dev/null 2>&1 &&
272+
git --git-dir="$1" --work-tree="$1" ls-files --stage >/dev/null 2>&1
273+
}
274+
drop_damaged_gitdirs() {
275+
local d
276+
for d in "$1"/*/; do
277+
d="${d%/}"
278+
[ -d "${d}" ] && [ ! -L "${d}" ] || continue
279+
if ! is_gitdir "${d}"; then
280+
drop_damaged_gitdirs "${d}"
281+
elif gitdir_ok "${d}"; then
282+
drop_damaged_gitdirs "${d}/modules"
283+
else
284+
echo "[git-repair] Removing damaged git dir: ${d#"${repo_dir}/"}"
285+
rm -rf "${d:?}" || true
286+
fi
287+
done
288+
}
289+
# `submodule update` re-clones only into an empty worktree, so a
290+
# worktree whose git dir is gone or partial, or whose .git file
291+
# does not resolve, is removed together with that git dir.
292+
drop_orphaned_worktrees() {
293+
local entry name path wt gd
294+
[ -f "$1/.gitmodules" ] || return 0
295+
while IFS= read -r -d '' entry; do
296+
name="${entry%%$'\n'*}"
297+
name="${name#submodule.}"
298+
name="${name%.path}"
299+
path="${entry#*$'\n'}"
300+
case "/${name}/ /${path}/" in *//*|*/./*|*/../*) continue ;; esac
301+
wt="$1/${path}"
302+
gd="$2/modules/${name}"
303+
if is_gitdir "${gd}" &&
304+
{ [ ! -f "${wt}/.git" ] || git -C "${wt}" rev-parse --git-dir >/dev/null 2>&1; }; then
305+
drop_orphaned_worktrees "${wt}" "${gd}"
306+
elif [ -e "${gd}" ] || [ -n "$(ls -A "${wt}" 2>/dev/null)" ]; then
307+
echo "[git-repair] Removing damaged submodule: ${wt#"${repo_dir}/"}"
308+
rm -rf "${gd:?}" "${wt:?}" || true
309+
fi
310+
done < <(git config -z -f "$1/.gitmodules" --get-regexp '^submodule\..*\.path$' 2>/dev/null)
311+
}
312+
drop_damaged_gitdirs "${repo_dir:?}/.git/modules"
313+
drop_orphaned_worktrees "${repo_dir:?}" "${repo_dir:?}/.git"
314+
fi
315+
246316
- name: Checkout code
247317
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
248318
with:

0 commit comments

Comments
 (0)