Skip to content

Commit 3bb4263

Browse files
committed
worker: drain retirement latch and hand-back retry budget
Plan 6 tasks 1 to 4 as one change set. - Shell test helper: shell argv tests bound their poll wait and narrow the no-/proc scan. - CI retry visibility: a retried test step reports its attempts. - Native exit naming: powershell_runner failures name the native exit code. - Drain retirement: unreached paths of a retiring drain are handed back without charging a retry, through a handBack hook that drainOnce supplies.
1 parent 6925940 commit 3bb4263

5 files changed

Lines changed: 113 additions & 10 deletions

File tree

‎src/worker.ts‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,9 @@ export function processDirtyBatch(
423423
remove: (absPath: string) => void = makeRemover(globalDbPath()),
424424
dir: string = dataDir(),
425425
requeue: (dir: string, absPath: string) => void = requeueDirtyPath,
426+
handBack: (dir: string, absPath: string) => void = (handBackDir, absPath) => {
427+
appendToDirtyQueue(handBackDir, absPath)
428+
},
426429
): number {
427430
const blockedRoots = loadConfig().worker.blocked_roots
428431
const ixCfg = loadConfig().indexing
@@ -435,7 +438,7 @@ export function processDirtyBatch(
435438
if (drainShouldHandBack()) {
436439
// No retry is charged, since nothing failed. Persisted before the caller clears the claimed file that named them, so a crash in between duplicates a path (its sha gate absorbs that) and never drops one.
437440
const unreached = paths.slice(i).filter((q) => q.length > 0)
438-
if (!appendToDirtyQueue(dir, ...unreached)) for (const q of unreached) requeue(dir, q)
441+
if (!appendToDirtyQueue(dir, ...unreached)) for (const q of unreached) handBack(dir, q)
439442
break
440443
}
441444
writeDrainHeartbeat(dir)
@@ -516,15 +519,20 @@ export function drainOnce(
516519
const deferredRequeues: string[] = []
517520
// Each deferred requeue is also written at once to a sidecar named like a stage (b) fallback claim, because the queue files that named these paths are deleted before the live append below: a kill, or a throw out of stage (b), in between would otherwise lose them. This cycle's stage (a) listed its files before the sidecar existed, so only a later cycle recovers it, and the deferral holds.
518521
const requeueSidecar = `${draining}.alt-requeue-${process.pid}-${Date.now()}`
519-
const requeueFn = (requeueDir: string, absPath: string): void => {
520-
if (!bumpAndCheckRetry(requeueDir, absPath)) return
522+
const deferAppend = (absPath: string): void => {
521523
deferredRequeues.push(absPath)
522524
try {
523525
appendDirtyQueueFile(requeueSidecar, [absPath])
524526
} catch {
525527
// The in-memory copy still reaches the live queue at the end of this cycle; only a kill before then loses it.
526528
}
527529
}
530+
const requeueFn = (requeueDir: string, absPath: string): void => {
531+
if (!bumpAndCheckRetry(requeueDir, absPath)) return
532+
deferAppend(absPath)
533+
}
534+
// Unreached paths of a retiring drain: deferred the same way, but never charged a retry, since nothing failed.
535+
const handBackFn = (_handBackDir: string, absPath: string): void => deferAppend(absPath)
528536

529537
// (a) Crash recovery: absorb any `.draining` (and `.draining.alt-*` fallback) files abandoned by a previous crashed or stuck drain. Multiple files can accumulate when a Windows sharing violation keeps the primary `.draining` name locked across cycles (see stage (b)'s fallback-claim comment) -- recover every one of them, not just the first, so a single stuck file can never starve the rest of the queue from ever draining.
530538
const liveDrainingFiles = listDrainingFiles(queuePath)
@@ -557,7 +565,7 @@ export function drainOnce(
557565
}
558566
// Only process this content if it was not already folded into a batch on a prior cycle (see unclearedDrainingSnapshots below). Without this guard, a draining file that outlives both cleanup attempts (e.g. a persistent Windows sharing violation) would be re-read and its paths reprocessed on every cycle.
559567
if (unclearedDrainingSnapshots.get(drainingFile) !== drainingSnapshotStamp(drainingFile, drainingContent)) {
560-
processed += processDirtyBatch(parseDirtyQueueLines(drainingContent), indexFn, removeFn, dir, requeueFn)
568+
processed += processDirtyBatch(parseDirtyQueueLines(drainingContent), indexFn, removeFn, dir, requeueFn, handBackFn)
561569
}
562570
// Only clear the recovered file now that its batch has been durably processed (or recognized above as already processed) -- never before -- so a crash partway through processDirtyBatch leaves the file in place for the next startup to recover, instead of deleting it up front and losing every path it named.
563571
try {
@@ -600,7 +608,7 @@ export function drainOnce(
600608
}
601609
if (readOk) {
602610
// Deliberately NOT wrapped in the try above: a throw from processDirtyBatch (e.g. the process crashing mid-batch) must propagate to the caller, not be swallowed as a "read failure", so the cleanup below never runs and the claimed file survives.
603-
processed += processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn)
611+
processed += processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn, handBackFn)
604612
// Re-read the claimed file ONE more time, right before deleting it, to catch a path appended by a writer that raced the claim-rename above. The rename-to-claim pattern assumes no other writer can touch the live queue's underlying file once it has been renamed to the claim target -- but that isn't guaranteed on Windows (FILE_SHARE_DELETE lets a concurrent open-for-append follow the same underlying file object across the rename) or on POSIX (a narrow rename/open race exists there too). A dirty path appended during that window would otherwise be silently deleted along with the rest of this cycle's already-processed content and never reindexed. Forward anything found beyond what was already processed back into the live queue so the next drain cycle picks it up.
605613
try {
606614
const recheck = fs.readFileSync(claimTarget, 'utf8')

‎tests/guards/critical_path_seam_coverage.test.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,21 +22,28 @@ const DECISIONS: Record<string, Decision> = {
2222
exempt: {
2323
reason: 'the only production caller is drainOnce, which passes its own index/remove/dir/requeue explicitly; these defaults are reached by direct-call tests only, and drainOnce\'s defaults (covered below) are what ships',
2424
file: 'src/worker.ts',
25-
marker: 'processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn)',
25+
marker: 'processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn, handBackFn)',
2626
},
2727
},
2828
'src/worker.ts::processDirtyBatch::remove': {
2929
exempt: {
3030
reason: 'same as processDirtyBatch::index -- drainOnce supplies it on every production call',
3131
file: 'src/worker.ts',
32-
marker: 'processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn)',
32+
marker: 'processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn, handBackFn)',
3333
},
3434
},
3535
'src/worker.ts::processDirtyBatch::requeue': {
3636
exempt: {
3737
reason: 'same as processDirtyBatch::index -- drainOnce supplies its own requeueFn so the deferred-requeue ordering holds',
3838
file: 'src/worker.ts',
39-
marker: 'processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn)',
39+
marker: 'processDirtyBatch(parseDirtyQueueLines(claimedContent), indexFn, removeFn, dir, requeueFn, handBackFn)',
40+
},
41+
},
42+
'src/worker.ts::processDirtyBatch::handBack': {
43+
exempt: {
44+
reason: 'same as processDirtyBatch::index -- drainOnce supplies it on every production call',
45+
file: 'src/worker.ts',
46+
marker: 'handBackFn)',
4047
},
4148
},
4249
'src/worker.ts::drainOnce::index': {

‎tests/shell_argv_helper.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -412,7 +412,7 @@ describe.skipIf(POSIX_SH === null)('shRunAll', () => {
412412
})
413413

414414
// Provenance: HAND-DERIVED from a macOS CI failure of the test above: macOS has no /proc, and its BSD `ps -e` lists the caller's own processes that have a terminal, with their environment, so a child started with no terminal was invisible to the kin scan, which also ran only where /proc exists. The branch is forced by making the /proc test fail, and `ps` is a function that answers as macOS does: `-wwAE` prints the owned parent with the token, `-e -o ...` prints nothing, and `-A -o ...` is the real listing.
415-
it('ends an owned child whose environment does not carry the token where there is no /proc and ps -e lists nothing', async () => {
415+
it('ends an owned child whose environment does not carry the token where there is no /proc and ps lists only the owned parent and the late child', async () => {
416416
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tg-shbsd-'))
417417
const file = (name: string): string => path.join(dir, name).replace(/\\/g, '/')
418418
// The /proc test is forced false, and the environ scan is narrowed to the two pid files, so the row does not pay the whole-table /proc grep under load; the kin listing the row tests is still the real fixture above.
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/** A path a retiring drain hands back was never tried, so handing it back must not use up its retry budget. Pinned in-thread: the daemon's pid file goes missing during the first batch's heartbeat, so the remaining paths are unreached, and the live queue path is a directory, so appending them to it fails and each takes the hand-back fallback. */
2+
import * as fs from 'node:fs'
3+
import * as os from 'node:os'
4+
import * as path from 'node:path'
5+
6+
import { afterEach, describe, expect, it, vi } from 'vitest'
7+
8+
import type * as Lifecycle from '../src/worker_lifecycle.js'
9+
import type * as Preflight from '../src/embed_preflight.js'
10+
11+
// A drained file would dispatch an embed call once the loop is past its drain; holding the model download keeps the loop from reaching one.
12+
vi.mock('../src/embed_preflight.js', async (importOriginal) => {
13+
const actual = await importOriginal<typeof Preflight>()
14+
return { ...actual, modelDownloadHeld: () => true }
15+
})
16+
17+
// The drain's per-path heartbeat runs after the retire decision for that path, so the first call marks the point where the daemon's pid file goes missing mid-batch.
18+
const hooks = vi.hoisted(() => ({ onFirstHeartbeat: null as null | (() => void) }))
19+
vi.mock('../src/worker_lifecycle.js', async (importOriginal) => {
20+
const actual = await importOriginal<typeof Lifecycle>()
21+
return {
22+
...actual,
23+
writeDrainHeartbeat: (dir: string, force?: boolean) => {
24+
const hook = hooks.onFirstHeartbeat
25+
hooks.onFirstHeartbeat = null
26+
hook?.()
27+
return actual.writeDrainHeartbeat(dir, force)
28+
},
29+
}
30+
})
31+
32+
import { closeDb, getDb } from '../src/db.js'
33+
import { dirtyQueuePathFor } from '../src/dirty_queue.js'
34+
import { runWorkerLoop } from '../src/worker.js'
35+
import { workerPidPath } from '../src/worker_lifecycle.js'
36+
37+
const dirs: string[] = []
38+
39+
function scratch(prefix: string): string {
40+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix))
41+
dirs.push(dir)
42+
return dir
43+
}
44+
45+
afterEach(() => {
46+
for (const d of dirs.splice(0)) {
47+
closeDb(path.join(d, 'global.db'))
48+
try {
49+
fs.rmSync(d, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 })
50+
} catch {
51+
// best-effort
52+
}
53+
}
54+
})
55+
56+
describe('a retiring drain hands its unreached paths back without charging a retry', () => {
57+
it('leaves the retry budget of every unreached path untouched', async () => {
58+
const dir = scratch('tg-handback-budget-')
59+
const repo = scratch('tg-handback-budget-repo-')
60+
const queued: string[] = []
61+
for (let i = 0; i < 3; i++) {
62+
const f = path.join(repo, `handback_${i}.ts`)
63+
fs.writeFileSync(f, `export const h${i} = ${i}\n`)
64+
queued.push(f)
65+
}
66+
const queuePath = dirtyQueuePathFor(dir)
67+
fs.mkdirSync(path.dirname(queuePath), { recursive: true })
68+
fs.writeFileSync(queuePath, `${queued.join('\n')}\n`)
69+
fs.writeFileSync(workerPidPath(dir), `${process.pid}\n`)
70+
71+
let retiredInDrain = 0
72+
hooks.onFirstHeartbeat = () => {
73+
retiredInDrain++
74+
fs.rmSync(workerPidPath(dir))
75+
// The live queue path cannot take an append, so the hand-back of the unreached paths falls back to one call per path.
76+
fs.mkdirSync(queuePath)
77+
// The retire probe is rate-limited, so wait past its window before the next path asks it again.
78+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 150)
79+
}
80+
81+
await runWorkerLoop(dir, 5, () => false, { pidFileGraceMs: 0, idleExitMs: 60_000 })
82+
fs.rmSync(queuePath, { recursive: true, force: true })
83+
84+
expect(retiredInDrain, 'the daemon did not retire during its first batch').toBe(1)
85+
const charged = getDb(path.join(dir, 'global.db')).prepare('SELECT COUNT(*) AS n FROM index_retries WHERE retry_count > 0').get() as { n: number }
86+
expect(charged.n, 'unreached paths were charged a transient retry').toBe(0)
87+
})
88+
})

‎tests/worker_retire_e2e.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ describe('the built daemon retires on its own', () => {
148148
observedGone(pid)
149149
expect(retiredAfterMs, `the daemon outlived its pid file by ${retiredAfterMs} ms`).toBeLessThanOrEqual(WORKER_STARTUP_GRACE_MS + 5000)
150150
},
151-
90000,
151+
60000,
152152
)
153153

154154
it(

0 commit comments

Comments
 (0)