|
| 1 | +#!/usr/bin/env node |
| 2 | +import { performance } from "node:perf_hooks"; |
| 3 | +import { parseFocusManifest, runIterateLoop } from "../dist/index.js"; |
| 4 | + |
| 5 | +// Load-testing harness for the AMS iterate-loop orchestrator (#5224): every existing iterate-loop test is |
| 6 | +// correctness-oriented (single attempt, fake driver returns instantly), so there is no signal today for how |
| 7 | +// runIterateLoop behaves when many tenants' attempts run concurrently against shared infra. This harness |
| 8 | +// reuses the SAME fake-driver injection seam iterate-loop.test.ts already exercises (a CodingAgentDriver whose |
| 9 | +// `run()` never spawns a real subprocess or spends API budget) but adds a configurable artificial per-iteration |
| 10 | +// delay, so the measured throughput reflects iterate-loop's own orchestration/scheduling overhead under |
| 11 | +// concurrency rather than a network call's latency. See docs/iterate-loop-load-test.md (#5224) for how to run |
| 12 | +// this and read the numbers, and issue #4913 for the parallel Worker-endpoint load-testing precedent. |
| 13 | + |
| 14 | +export const DEFAULT_CONCURRENCY_LEVELS = [1, 8, 32, 128]; |
| 15 | +export const DEFAULT_ATTEMPTS_PER_LEVEL = 32; |
| 16 | +export const DEFAULT_SIMULATED_DRIVER_LATENCY_MS = 15; |
| 17 | + |
| 18 | +const SYNTHETIC_ISSUE_NUMBER = 7; |
| 19 | + |
| 20 | +/** One open issue per synthetic tenant repo, matching that tenant's own `passesPredictedGate` linkage below -- |
| 21 | + * each tenant is a fully independent repo/contributor pair (`buildSelfReviewPredictedGateInput`'s own identity |
| 22 | + * fields), the same "multi-tenant-like" shape the issue's Problem section asks this harness to load-test. */ |
| 23 | +function buildReviewContext(tenantRepoFullName) { |
| 24 | + return { |
| 25 | + manifest: parseFocusManifest({ gate: { duplicates: "block", linkedIssue: "advisory" } }), |
| 26 | + repo: { fullName: tenantRepoFullName, owner: tenantRepoFullName.split("/")[0], name: tenantRepoFullName.split("/")[1], isInstalled: true, isRegistered: true, isPrivate: false }, |
| 27 | + issues: [{ repoFullName: tenantRepoFullName, number: SYNTHETIC_ISSUE_NUMBER, title: "Synthetic load-test issue", state: "open", labels: [], linkedPrs: [] }], |
| 28 | + pullRequests: [], |
| 29 | + }; |
| 30 | +} |
| 31 | + |
| 32 | +/** A `CodingAgentDriver` (coding-agent-driver.ts) that never spawns a real subprocess or spends API budget -- |
| 33 | + * it resolves after `latencyMs` (simulating the wall-clock an iteration of a real coding-agent invocation |
| 34 | + * would take) with a scripted, always-passing result. `latencyMs` uses a real `setTimeout`, not a busy-loop, so |
| 35 | + * concurrent attempts genuinely interleave on the event loop the way concurrent live attempts would. */ |
| 36 | +export function buildFakeLoadTestDriver(latencyMs) { |
| 37 | + return { |
| 38 | + async run(task) { |
| 39 | + await new Promise((resolve) => setTimeout(resolve, latencyMs)); |
| 40 | + return { ok: true, changedFiles: [`src/${task.attemptId}.ts`], summary: `synthetic load-test change for ${task.attemptId}`, turnsUsed: 1 }; |
| 41 | + }, |
| 42 | + }; |
| 43 | +} |
| 44 | + |
| 45 | +const NOOP_SLOP_ASSESSMENT = { slopRisk: 0, band: "clean", findings: [] }; |
| 46 | + |
| 47 | +/** One simulated tenant attempt: a distinct `repoFullName`/`contributorLogin` per `tenantIndex`, a linked open |
| 48 | + * issue that matches on the first iteration, so every attempt hands off in exactly one iteration -- isolating |
| 49 | + * the measurement to iterate-loop's own per-attempt orchestration overhead rather than varying iteration counts |
| 50 | + * across runs. */ |
| 51 | +async function runOneAttempt(tenantIndex, driver) { |
| 52 | + const repoFullName = `load-test-tenant-${tenantIndex}/repo`; |
| 53 | + const attemptId = `attempt-${tenantIndex}`; |
| 54 | + const input = { |
| 55 | + attemptId, |
| 56 | + workingDirectory: `/tmp/${attemptId}`, |
| 57 | + acceptanceCriteriaPath: `/tmp/${attemptId}/acceptance-criteria.json`, |
| 58 | + instructions: "Synthetic load-test instructions", |
| 59 | + mode: "live", |
| 60 | + maxIterations: 3, |
| 61 | + maxTurnsPerIteration: 20, |
| 62 | + repoFullName, |
| 63 | + contributorLogin: `miner-${tenantIndex}`, |
| 64 | + title: "Synthetic load-test attempt", |
| 65 | + body: `Closes #${SYNTHETIC_ISSUE_NUMBER}`, |
| 66 | + linkedIssues: [SYNTHETIC_ISSUE_NUMBER], |
| 67 | + reviewContext: buildReviewContext(repoFullName), |
| 68 | + rejectionSignaled: false, |
| 69 | + }; |
| 70 | + const deps = { |
| 71 | + driver, |
| 72 | + runSlopAssessment: () => NOOP_SLOP_ASSESSMENT, |
| 73 | + appendAttemptLogEvent: () => {}, |
| 74 | + }; |
| 75 | + const start = performance.now(); |
| 76 | + const result = await runIterateLoop(input, deps); |
| 77 | + return { elapsedMs: performance.now() - start, result }; |
| 78 | +} |
| 79 | + |
| 80 | +/** |
| 81 | + * Run `attemptCount` simulated tenant attempts concurrently (`Promise.all`, all started in the same tick) against |
| 82 | + * one shared fake driver, and report the aggregate wall time plus derived throughput. Every attempt is expected |
| 83 | + * to hand off after its first iteration (see {@link runOneAttempt}) -- a non-`"handoff"` outcome or a driver |
| 84 | + * error would silently understate real concurrent load, so both are counted and surfaced rather than ignored. |
| 85 | + */ |
| 86 | +export async function runConcurrencyLevel(concurrency, options = {}) { |
| 87 | + const attemptCount = options.attemptCount ?? DEFAULT_ATTEMPTS_PER_LEVEL; |
| 88 | + const latencyMs = options.latencyMs ?? DEFAULT_SIMULATED_DRIVER_LATENCY_MS; |
| 89 | + const driver = buildFakeLoadTestDriver(latencyMs); |
| 90 | + |
| 91 | + const start = performance.now(); |
| 92 | + const outcomes = []; |
| 93 | + for (let batchStart = 0; batchStart < attemptCount; batchStart += concurrency) { |
| 94 | + const batchSize = Math.min(concurrency, attemptCount - batchStart); |
| 95 | + const batch = await Promise.all( |
| 96 | + Array.from({ length: batchSize }, (_unused, offset) => runOneAttempt(batchStart + offset, driver)), |
| 97 | + ); |
| 98 | + outcomes.push(...batch); |
| 99 | + } |
| 100 | + const wallMs = performance.now() - start; |
| 101 | + const handoffCount = outcomes.filter((o) => o.result.outcome === "handoff").length; |
| 102 | + |
| 103 | + return { |
| 104 | + concurrency, |
| 105 | + attemptCount, |
| 106 | + latencyMs, |
| 107 | + wallMs, |
| 108 | + handoffCount, |
| 109 | + attemptsPerSecond: attemptCount / (wallMs / 1000), |
| 110 | + }; |
| 111 | +} |
| 112 | + |
| 113 | +/** Run every concurrency level in `levels` in sequence (never overlapping each other), so one level's |
| 114 | + * scheduling contention never bleeds into the next level's measurement. */ |
| 115 | +export async function runLoadTest(options = {}) { |
| 116 | + const levels = options.levels ?? DEFAULT_CONCURRENCY_LEVELS; |
| 117 | + const results = []; |
| 118 | + for (const concurrency of levels) { |
| 119 | + results.push(await runConcurrencyLevel(concurrency, options)); |
| 120 | + } |
| 121 | + return results; |
| 122 | +} |
| 123 | + |
| 124 | +/** Render load-test results as a stable, greppable text report (no locale-dependent number formatting). */ |
| 125 | +export function formatLoadTestReport(results) { |
| 126 | + const lines = ["iterate-loop load test", ""]; |
| 127 | + for (const r of results) { |
| 128 | + lines.push( |
| 129 | + `concurrency=${r.concurrency}: ${r.wallMs.toFixed(2)}ms wall for ${r.attemptCount} attempts, ` + |
| 130 | + `${Math.round(r.attemptsPerSecond)} attempts/sec, ${r.handoffCount}/${r.attemptCount} handed off ` + |
| 131 | + `(simulated driver latency ${r.latencyMs}ms)`, |
| 132 | + ); |
| 133 | + } |
| 134 | + return lines.join("\n"); |
| 135 | +} |
| 136 | + |
| 137 | +async function main() { |
| 138 | + const results = await runLoadTest(); |
| 139 | + console.log(formatLoadTestReport(results)); |
| 140 | +} |
| 141 | + |
| 142 | +if (import.meta.url === `file://${process.argv[1]}`) { |
| 143 | + main(); |
| 144 | +} |
0 commit comments