Skip to content

Commit c669164

Browse files
adibarraadibarra
committed
fix(ci): drop refs to missing objects before checkout
The gb200-nv_17 test run with the first repair still failed: `git checkout --force <sha>` died with "bad object refs/remotes/origin/vllm/dsv41flash-b200-image-bump". Checkout walks every ref to warn about commits left behind, and a fetch inside a submodule walks every ref when its pin moves, so one ref whose object is gone fails every job. The repair step now deletes refs whose objects are missing, in the top-level repository and in every module git dir, with one for-each-ref, one batched cat-file --batch-check and one update-ref --stdin per repository. A git dir whose refs cannot be listed counts as damaged. The top-level .git is removed, so checkout re-clones it, when HEAD's commit, trees or blobs are missing or its refs cannot be listed; profile.yml checks out with clean: false and could not recover from that before. Dropping a module git dir now also removes the worktree it serves, so a worktree left by a module that is no longer in .gitmodules cannot keep pointing at a missing git dir.
1 parent e123d80 commit c669164

5 files changed

Lines changed: 205 additions & 55 deletions

File tree

‎.github/workflows/benchmark-multinode-tmpl.yml‎

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -233,13 +233,14 @@ jobs:
233233
- name: Repair stale git state (pre-run)
234234
shell: bash
235235
run: |
236-
# Interrupted jobs leave stale locks and damaged submodule git state
237-
# in the persistent workspace. actions/checkout only clears the
238-
# top-level index.lock/shallow.lock and recreates the repo when the
239-
# top-level clean/reset fails, yet every checkout also runs
240-
# `git submodule foreach --recursive`, which dies on any damaged
241-
# submodule. One job runs per runner, so every lock here is stale.
242-
# Keep identical in benchmark-tmpl.yml, benchmark-multinode-tmpl.yml,
236+
# Interrupted jobs leave stale locks and damaged git state in the
237+
# persistent workspace. actions/checkout only clears the top-level
238+
# index.lock/shallow.lock and recreates the repo when the top-level
239+
# clean/reset fails, yet every checkout also runs
240+
# `git submodule foreach --recursive` and walks every ref, so a
241+
# damaged submodule or a ref to a missing object fails the job. One
242+
# job runs per runner, so every lock here is stale. Keep identical in
243+
# benchmark-tmpl.yml, benchmark-multinode-tmpl.yml,
243244
# operatorx-sweep.yml, profile.yml and speedbench-al.yml.
244245
repo_dir="${GITHUB_WORKSPACE}"
245246
if [ -d "${repo_dir}/.git" ]; then
@@ -248,6 +249,19 @@ jobs:
248249
rm -f "${lock}" || true
249250
done < <(find "${repo_dir}/.git" -path '*/objects/[0-9a-f][0-9a-f]' -prune \
250251
-o -name '*.lock' -type f -print0)
252+
# Checkout fetches by SHA, so refs to missing objects are
253+
# disposable; a repo whose refs cannot be listed is damaged.
254+
drop_dangling_refs() {
255+
local refs dangling
256+
refs=$(git --git-dir="$1" --work-tree="$1" for-each-ref --format='%(objectname) %(refname)' 2>/dev/null) || return 1
257+
[ -n "${refs}" ] || return 0
258+
mapfile -t dangling < <(paste -d ' ' <(cut -d ' ' -f 2 <<<"${refs}") \
259+
<(cut -d ' ' -f 1 <<<"${refs}" | git --git-dir="$1" --work-tree="$1" cat-file --batch-check 2>/dev/null) |
260+
awk '$NF == "missing" { print $1 }')
261+
[ "${#dangling[@]}" -gt 0 ] || return 0
262+
echo "[git-repair] Removing refs to missing objects in ${1#"${repo_dir}/"}: ${dangling[*]}"
263+
printf 'delete %s\n' "${dangling[@]}" | git --git-dir="$1" --work-tree="$1" update-ref --no-deref --stdin
264+
}
251265
# Module git dirs sit at any depth below .git/modules; names may
252266
# contain '/', so dirs without HEAD/config/objects/refs only group
253267
# them. Every checkout reads each module's remote.origin.url, HEAD
@@ -259,16 +273,24 @@ jobs:
259273
git --git-dir="$1" --work-tree="$1" ls-files --stage >/dev/null 2>&1
260274
}
261275
drop_damaged_gitdirs() {
262-
local d
276+
local d wt root
263277
for d in "$1"/*/; do
264278
d="${d%/}"
265279
[ -d "${d}" ] && [ ! -L "${d}" ] || continue
266280
if ! is_gitdir "${d}"; then
267281
drop_damaged_gitdirs "${d}"
268-
elif gitdir_ok "${d}"; then
282+
elif gitdir_ok "${d}" && drop_dangling_refs "${d}"; then
269283
drop_damaged_gitdirs "${d}/modules"
270284
else
271285
echo "[git-repair] Removing damaged git dir: ${d#"${repo_dir}/"}"
286+
# Its worktree, even one no longer in .gitmodules, would keep
287+
# a .git file pointing at the removed git dir.
288+
if wt=$(git config -f "${d}/config" core.worktree 2>/dev/null) &&
289+
wt=$(cd "${d}" && cd "${wt}" && pwd -P) && root=$(cd "${repo_dir}" && pwd -P) &&
290+
[ -f "${wt}/.git" ] && [ "${wt#"${root}"/}" != "${wt}" ]; then
291+
echo "[git-repair] Removing its worktree: ${wt#"${root}"/}"
292+
rm -rf "${wt:?}" || true
293+
fi
272294
rm -rf "${d:?}" || true
273295
fi
274296
done
@@ -296,8 +318,16 @@ jobs:
296318
fi
297319
done < <(git config -z -f "$1/.gitmodules" --get-regexp '^submodule\..*\.path$' 2>/dev/null)
298320
}
299-
drop_damaged_gitdirs "${repo_dir:?}/.git/modules"
300-
drop_orphaned_worktrees "${repo_dir:?}" "${repo_dir:?}/.git"
321+
# A top-level repo with incomplete HEAD objects or unlistable refs
322+
# is removed, so checkout re-clones it instead of failing.
323+
if git --git-dir="${repo_dir}/.git" --work-tree="${repo_dir}" rev-list --objects --no-walk HEAD >/dev/null 2>&1 &&
324+
drop_dangling_refs "${repo_dir}/.git"; then
325+
drop_damaged_gitdirs "${repo_dir:?}/.git/modules"
326+
drop_orphaned_worktrees "${repo_dir:?}" "${repo_dir:?}/.git"
327+
else
328+
echo "[git-repair] Removing damaged repository: .git"
329+
rm -rf "${repo_dir:?}/.git" || true
330+
fi
301331
fi
302332
303333
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

‎.github/workflows/benchmark-tmpl.yml‎

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -190,13 +190,14 @@ jobs:
190190
- name: Repair stale git state (pre-run)
191191
shell: bash
192192
run: |
193-
# Interrupted jobs leave stale locks and damaged submodule git state
194-
# in the persistent workspace. actions/checkout only clears the
195-
# top-level index.lock/shallow.lock and recreates the repo when the
196-
# top-level clean/reset fails, yet every checkout also runs
197-
# `git submodule foreach --recursive`, which dies on any damaged
198-
# submodule. One job runs per runner, so every lock here is stale.
199-
# Keep identical in benchmark-tmpl.yml, benchmark-multinode-tmpl.yml,
193+
# Interrupted jobs leave stale locks and damaged git state in the
194+
# persistent workspace. actions/checkout only clears the top-level
195+
# index.lock/shallow.lock and recreates the repo when the top-level
196+
# clean/reset fails, yet every checkout also runs
197+
# `git submodule foreach --recursive` and walks every ref, so a
198+
# damaged submodule or a ref to a missing object fails the job. One
199+
# job runs per runner, so every lock here is stale. Keep identical in
200+
# benchmark-tmpl.yml, benchmark-multinode-tmpl.yml,
200201
# operatorx-sweep.yml, profile.yml and speedbench-al.yml.
201202
repo_dir="${GITHUB_WORKSPACE}"
202203
if [ -d "${repo_dir}/.git" ]; then
@@ -205,6 +206,19 @@ jobs:
205206
rm -f "${lock}" || true
206207
done < <(find "${repo_dir}/.git" -path '*/objects/[0-9a-f][0-9a-f]' -prune \
207208
-o -name '*.lock' -type f -print0)
209+
# Checkout fetches by SHA, so refs to missing objects are
210+
# disposable; a repo whose refs cannot be listed is damaged.
211+
drop_dangling_refs() {
212+
local refs dangling
213+
refs=$(git --git-dir="$1" --work-tree="$1" for-each-ref --format='%(objectname) %(refname)' 2>/dev/null) || return 1
214+
[ -n "${refs}" ] || return 0
215+
mapfile -t dangling < <(paste -d ' ' <(cut -d ' ' -f 2 <<<"${refs}") \
216+
<(cut -d ' ' -f 1 <<<"${refs}" | git --git-dir="$1" --work-tree="$1" cat-file --batch-check 2>/dev/null) |
217+
awk '$NF == "missing" { print $1 }')
218+
[ "${#dangling[@]}" -gt 0 ] || return 0
219+
echo "[git-repair] Removing refs to missing objects in ${1#"${repo_dir}/"}: ${dangling[*]}"
220+
printf 'delete %s\n' "${dangling[@]}" | git --git-dir="$1" --work-tree="$1" update-ref --no-deref --stdin
221+
}
208222
# Module git dirs sit at any depth below .git/modules; names may
209223
# contain '/', so dirs without HEAD/config/objects/refs only group
210224
# them. Every checkout reads each module's remote.origin.url, HEAD
@@ -216,16 +230,24 @@ jobs:
216230
git --git-dir="$1" --work-tree="$1" ls-files --stage >/dev/null 2>&1
217231
}
218232
drop_damaged_gitdirs() {
219-
local d
233+
local d wt root
220234
for d in "$1"/*/; do
221235
d="${d%/}"
222236
[ -d "${d}" ] && [ ! -L "${d}" ] || continue
223237
if ! is_gitdir "${d}"; then
224238
drop_damaged_gitdirs "${d}"
225-
elif gitdir_ok "${d}"; then
239+
elif gitdir_ok "${d}" && drop_dangling_refs "${d}"; then
226240
drop_damaged_gitdirs "${d}/modules"
227241
else
228242
echo "[git-repair] Removing damaged git dir: ${d#"${repo_dir}/"}"
243+
# Its worktree, even one no longer in .gitmodules, would keep
244+
# a .git file pointing at the removed git dir.
245+
if wt=$(git config -f "${d}/config" core.worktree 2>/dev/null) &&
246+
wt=$(cd "${d}" && cd "${wt}" && pwd -P) && root=$(cd "${repo_dir}" && pwd -P) &&
247+
[ -f "${wt}/.git" ] && [ "${wt#"${root}"/}" != "${wt}" ]; then
248+
echo "[git-repair] Removing its worktree: ${wt#"${root}"/}"
249+
rm -rf "${wt:?}" || true
250+
fi
229251
rm -rf "${d:?}" || true
230252
fi
231253
done
@@ -253,8 +275,16 @@ jobs:
253275
fi
254276
done < <(git config -z -f "$1/.gitmodules" --get-regexp '^submodule\..*\.path$' 2>/dev/null)
255277
}
256-
drop_damaged_gitdirs "${repo_dir:?}/.git/modules"
257-
drop_orphaned_worktrees "${repo_dir:?}" "${repo_dir:?}/.git"
278+
# A top-level repo with incomplete HEAD objects or unlistable refs
279+
# is removed, so checkout re-clones it instead of failing.
280+
if git --git-dir="${repo_dir}/.git" --work-tree="${repo_dir}" rev-list --objects --no-walk HEAD >/dev/null 2>&1 &&
281+
drop_dangling_refs "${repo_dir}/.git"; then
282+
drop_damaged_gitdirs "${repo_dir:?}/.git/modules"
283+
drop_orphaned_worktrees "${repo_dir:?}" "${repo_dir:?}/.git"
284+
else
285+
echo "[git-repair] Removing damaged repository: .git"
286+
rm -rf "${repo_dir:?}/.git" || true
287+
fi
258288
fi
259289
260290
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

‎.github/workflows/operatorx-sweep.yml‎

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -121,13 +121,14 @@ jobs:
121121
- name: Repair stale git state (pre-run)
122122
shell: bash
123123
run: |
124-
# Interrupted jobs leave stale locks and damaged submodule git state
125-
# in the persistent workspace. actions/checkout only clears the
126-
# top-level index.lock/shallow.lock and recreates the repo when the
127-
# top-level clean/reset fails, yet every checkout also runs
128-
# `git submodule foreach --recursive`, which dies on any damaged
129-
# submodule. One job runs per runner, so every lock here is stale.
130-
# Keep identical in benchmark-tmpl.yml, benchmark-multinode-tmpl.yml,
124+
# Interrupted jobs leave stale locks and damaged git state in the
125+
# persistent workspace. actions/checkout only clears the top-level
126+
# index.lock/shallow.lock and recreates the repo when the top-level
127+
# clean/reset fails, yet every checkout also runs
128+
# `git submodule foreach --recursive` and walks every ref, so a
129+
# damaged submodule or a ref to a missing object fails the job. One
130+
# job runs per runner, so every lock here is stale. Keep identical in
131+
# benchmark-tmpl.yml, benchmark-multinode-tmpl.yml,
131132
# operatorx-sweep.yml, profile.yml and speedbench-al.yml.
132133
repo_dir="${GITHUB_WORKSPACE}"
133134
if [ -d "${repo_dir}/.git" ]; then
@@ -136,6 +137,19 @@ jobs:
136137
rm -f "${lock}" || true
137138
done < <(find "${repo_dir}/.git" -path '*/objects/[0-9a-f][0-9a-f]' -prune \
138139
-o -name '*.lock' -type f -print0)
140+
# Checkout fetches by SHA, so refs to missing objects are
141+
# disposable; a repo whose refs cannot be listed is damaged.
142+
drop_dangling_refs() {
143+
local refs dangling
144+
refs=$(git --git-dir="$1" --work-tree="$1" for-each-ref --format='%(objectname) %(refname)' 2>/dev/null) || return 1
145+
[ -n "${refs}" ] || return 0
146+
mapfile -t dangling < <(paste -d ' ' <(cut -d ' ' -f 2 <<<"${refs}") \
147+
<(cut -d ' ' -f 1 <<<"${refs}" | git --git-dir="$1" --work-tree="$1" cat-file --batch-check 2>/dev/null) |
148+
awk '$NF == "missing" { print $1 }')
149+
[ "${#dangling[@]}" -gt 0 ] || return 0
150+
echo "[git-repair] Removing refs to missing objects in ${1#"${repo_dir}/"}: ${dangling[*]}"
151+
printf 'delete %s\n' "${dangling[@]}" | git --git-dir="$1" --work-tree="$1" update-ref --no-deref --stdin
152+
}
139153
# Module git dirs sit at any depth below .git/modules; names may
140154
# contain '/', so dirs without HEAD/config/objects/refs only group
141155
# them. Every checkout reads each module's remote.origin.url, HEAD
@@ -147,16 +161,24 @@ jobs:
147161
git --git-dir="$1" --work-tree="$1" ls-files --stage >/dev/null 2>&1
148162
}
149163
drop_damaged_gitdirs() {
150-
local d
164+
local d wt root
151165
for d in "$1"/*/; do
152166
d="${d%/}"
153167
[ -d "${d}" ] && [ ! -L "${d}" ] || continue
154168
if ! is_gitdir "${d}"; then
155169
drop_damaged_gitdirs "${d}"
156-
elif gitdir_ok "${d}"; then
170+
elif gitdir_ok "${d}" && drop_dangling_refs "${d}"; then
157171
drop_damaged_gitdirs "${d}/modules"
158172
else
159173
echo "[git-repair] Removing damaged git dir: ${d#"${repo_dir}/"}"
174+
# Its worktree, even one no longer in .gitmodules, would keep
175+
# a .git file pointing at the removed git dir.
176+
if wt=$(git config -f "${d}/config" core.worktree 2>/dev/null) &&
177+
wt=$(cd "${d}" && cd "${wt}" && pwd -P) && root=$(cd "${repo_dir}" && pwd -P) &&
178+
[ -f "${wt}/.git" ] && [ "${wt#"${root}"/}" != "${wt}" ]; then
179+
echo "[git-repair] Removing its worktree: ${wt#"${root}"/}"
180+
rm -rf "${wt:?}" || true
181+
fi
160182
rm -rf "${d:?}" || true
161183
fi
162184
done
@@ -184,8 +206,16 @@ jobs:
184206
fi
185207
done < <(git config -z -f "$1/.gitmodules" --get-regexp '^submodule\..*\.path$' 2>/dev/null)
186208
}
187-
drop_damaged_gitdirs "${repo_dir:?}/.git/modules"
188-
drop_orphaned_worktrees "${repo_dir:?}" "${repo_dir:?}/.git"
209+
# A top-level repo with incomplete HEAD objects or unlistable refs
210+
# is removed, so checkout re-clones it instead of failing.
211+
if git --git-dir="${repo_dir}/.git" --work-tree="${repo_dir}" rev-list --objects --no-walk HEAD >/dev/null 2>&1 &&
212+
drop_dangling_refs "${repo_dir}/.git"; then
213+
drop_damaged_gitdirs "${repo_dir:?}/.git/modules"
214+
drop_orphaned_worktrees "${repo_dir:?}" "${repo_dir:?}/.git"
215+
else
216+
echo "[git-repair] Removing damaged repository: .git"
217+
rm -rf "${repo_dir:?}/.git" || true
218+
fi
189219
fi
190220
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
191221
with:

0 commit comments

Comments
 (0)