|
| 1 | +import { |
| 2 | + DEFAULT_AMS_POLICY_SPEC, |
| 3 | + DEFAULT_WRITE_RATE_LIMIT_POLICIES, |
| 4 | + evaluateGovernorCaps, |
| 5 | + evaluateLocalRateLimit, |
| 6 | + type GovernorCapUsage, |
| 7 | + type LocalRateBucket, |
| 8 | + type LocalRateLimitDecision, |
| 9 | + type WriteRateLimitBucketStore, |
| 10 | +} from "@loopover/engine"; |
| 11 | +import { openGovernorState } from "./governor-state.js"; |
| 12 | +import type { GovernorRateLimitState, GovernorState } from "./governor-state.js"; |
| 13 | +import { argsWantJson, describeCliError, reportCliFailure } from "./cli-error.js"; |
| 14 | + |
| 15 | +// `governor metrics` (#5187): render the governor's persisted rate-limit + cap-usage state (#5134, |
| 16 | +// governor-state.js) as Prometheus text-exposition, so an operator's Alertmanager can page on rate-limit/ |
| 17 | +// budget pressure without hand-rolling a scrape. Strictly read-only, mirroring queue-cli.js's `queue metrics` |
| 18 | +// (#5186) and event-ledger-cli.js's `ledger metrics` (#4841): opens the local governor-state store, composes |
| 19 | +// its EXISTING loadRateLimitState()/loadCapUsage() with the engine's already-exported PURE calculators |
| 20 | +// (evaluateLocalRateLimit, evaluateGovernorCaps) against the SAME defaults the production loop (loop-cli.js) |
| 21 | +// already falls back to when no `.loopover-ams.yml` override is configured (DEFAULT_WRITE_RATE_LIMIT_POLICIES, |
| 22 | +// DEFAULT_AMS_POLICY_SPEC.capLimits) -- it never invents a threshold of its own, and it does not gate, retry, |
| 23 | +// mutate, or otherwise touch governor decision logic (governor-chokepoint.js/governor-chokepoint-persisted.js |
| 24 | +// are completely untouched by this file). |
| 25 | +// |
| 26 | +// capLimits is intentionally NOT read per-repo: governor-state.js's capUsage row is a single global scalar (a |
| 27 | +// run-scoped cumulative counter, not indexed by repo -- see governor-state.js's own header comment), so a |
| 28 | +// per-repo capLimits override from a resolved `.loopover-miner.yml` has no matching per-repo usage row to |
| 29 | +// pair it with here. Using the fleet-wide DEFAULT_AMS_POLICY_SPEC.capLimits is the same approximation |
| 30 | +// loop-cli.js itself already makes for any repo without its own override. |
| 31 | + |
| 32 | +const GOVERNOR_METRICS_USAGE = "Usage: loopover-miner governor metrics"; |
| 33 | + |
| 34 | +export const GOVERNOR_RATE_LIMIT_REMAINING_RATIO = "loopover_miner_governor_rate_limit_remaining_ratio"; |
| 35 | +export const GOVERNOR_CAP_USAGE_RATIO = "loopover_miner_governor_cap_usage_ratio"; |
| 36 | + |
| 37 | +export type GovernorMetricsCliOptions = { |
| 38 | + openGovernorState?: () => GovernorState; |
| 39 | + nowMs?: number; |
| 40 | +}; |
| 41 | + |
| 42 | +type RateLimitMetricRow = { |
| 43 | + scope: string; |
| 44 | + actionClass: string; |
| 45 | + repoFullName: string; |
| 46 | + ratio: number; |
| 47 | +}; |
| 48 | + |
| 49 | +type CapUsageMetricRow = { |
| 50 | + dimension: string; |
| 51 | + ratio: number; |
| 52 | +}; |
| 53 | + |
| 54 | +/** HELP-text escaping — backslash + newline (mirrors miner-prediction-metrics.ts's escapeHelpText). */ |
| 55 | +function escapeMetricsHelpText(help: string): string { |
| 56 | + return help.replace(/\\/g, "\\\\").replace(/\n/g, "\\n"); |
| 57 | +} |
| 58 | + |
| 59 | +/** Prometheus label-value escaping — backslash, double-quote, newline (mirrors event-ledger-cli.js's |
| 60 | + * escapeLabelValue). */ |
| 61 | +function escapeLabelValue(value: string): string { |
| 62 | + return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n"); |
| 63 | +} |
| 64 | + |
| 65 | +/** buckets.perRepo is keyed by writeRateLimitRepoKey(actionClass, repoFullName) = "actionClass:repoFullName" |
| 66 | + * (write-rate-limit.ts). actionClass is a fixed identifier (never contains ":"), so splitting on the FIRST |
| 67 | + * colon recovers both parts even though repoFullName itself contains a "/". */ |
| 68 | +function splitPerRepoKey(key: string): { actionClass: string; repoFullName: string } { |
| 69 | + const separatorIndex = key.indexOf(":"); |
| 70 | + if (separatorIndex === -1) return { actionClass: key, repoFullName: "" }; |
| 71 | + return { actionClass: key.slice(0, separatorIndex), repoFullName: key.slice(separatorIndex + 1) }; |
| 72 | +} |
| 73 | + |
| 74 | +// evaluateLocalRateLimit's own `remaining` field answers "how many MORE writes are allowed AFTER one more write |
| 75 | +// right now" (rate-limit.ts: `remaining = allowed ? limit - effectiveCount - 1 : 0`) -- it is NOT current |
| 76 | +// headroom. At count=2/limit=3 that field is already 0, identical to a fully exhausted count=3/limit=3 bucket, |
| 77 | +// even though the count=2 bucket still has one write available. Recover true current headroom algebraically |
| 78 | +// instead: when allowed, decision.remaining + 1 is exactly limit - effectiveCount (undo the "-1 for this next |
| 79 | +// write" the decision already applied); when not allowed, headroom is 0. Every actionClass this loop reaches |
| 80 | +// has already passed the DEFAULT_WRITE_RATE_LIMIT_POLICIES lookup above, so decision.limit is always one of the |
| 81 | +// frozen, non-zero policy limits -- no zero-limit guard needed. |
| 82 | +function remainingRatio(decision: LocalRateLimitDecision): number { |
| 83 | + const headroom = decision.allowed ? decision.remaining + 1 : 0; |
| 84 | + return headroom / decision.limit; |
| 85 | +} |
| 86 | + |
| 87 | +function collectRateLimitRows(buckets: WriteRateLimitBucketStore, nowMs: number): RateLimitMetricRow[] { |
| 88 | + const rows: RateLimitMetricRow[] = []; |
| 89 | + for (const [actionClass, bucket] of Object.entries(buckets.global) as [string, LocalRateBucket][]) { |
| 90 | + const config = DEFAULT_WRITE_RATE_LIMIT_POLICIES.global[actionClass]; |
| 91 | + if (!config) continue; |
| 92 | + rows.push({ |
| 93 | + scope: "global", |
| 94 | + actionClass, |
| 95 | + repoFullName: "", |
| 96 | + ratio: remainingRatio(evaluateLocalRateLimit(bucket, config, nowMs)), |
| 97 | + }); |
| 98 | + } |
| 99 | + for (const [key, bucket] of Object.entries(buckets.perRepo) as [string, LocalRateBucket][]) { |
| 100 | + const { actionClass, repoFullName } = splitPerRepoKey(key); |
| 101 | + const config = DEFAULT_WRITE_RATE_LIMIT_POLICIES.perRepo[actionClass]; |
| 102 | + if (!config) continue; |
| 103 | + rows.push({ |
| 104 | + scope: "per_repo", |
| 105 | + actionClass, |
| 106 | + repoFullName, |
| 107 | + ratio: remainingRatio(evaluateLocalRateLimit(bucket, config, nowMs)), |
| 108 | + }); |
| 109 | + } |
| 110 | + rows.sort((a, b) => { |
| 111 | + if (a.scope !== b.scope) return a.scope.localeCompare(b.scope); |
| 112 | + if (a.actionClass !== b.actionClass) return a.actionClass.localeCompare(b.actionClass); |
| 113 | + return a.repoFullName.localeCompare(b.repoFullName); |
| 114 | + }); |
| 115 | + return rows; |
| 116 | +} |
| 117 | + |
| 118 | +// DEFAULT_AMS_POLICY_SPEC.capLimits is a frozen, non-zero constant for every dimension -- no zero-limit guard |
| 119 | +// needed, mirroring remainingRatio()'s reasoning above. |
| 120 | +function collectCapUsageRows(capUsage: GovernorCapUsage): CapUsageMetricRow[] { |
| 121 | + const report = evaluateGovernorCaps(capUsage, DEFAULT_AMS_POLICY_SPEC.capLimits); |
| 122 | + return [ |
| 123 | + { dimension: "budget", dimensionReport: report.budget }, |
| 124 | + { dimension: "turns", dimensionReport: report.turns }, |
| 125 | + { dimension: "elapsed_ms", dimensionReport: report.termination }, |
| 126 | + ].map(({ dimension, dimensionReport }) => ({ |
| 127 | + dimension, |
| 128 | + ratio: dimensionReport.used / dimensionReport.limit, |
| 129 | + })); |
| 130 | +} |
| 131 | + |
| 132 | +export function renderGovernorMetrics( |
| 133 | + rateLimitState: GovernorRateLimitState, |
| 134 | + capUsage: GovernorCapUsage, |
| 135 | + nowMs: number, |
| 136 | +): string { |
| 137 | + const rateLimitRows = collectRateLimitRows(rateLimitState.buckets, nowMs); |
| 138 | + const capRows = collectCapUsageRows(capUsage); |
| 139 | + |
| 140 | + const lines = [ |
| 141 | + `# HELP ${GOVERNOR_RATE_LIMIT_REMAINING_RATIO} ${escapeMetricsHelpText( |
| 142 | + "Remaining headroom in the governor's current write-rate-limit window, as a fraction of the configured limit (1 = empty bucket, 0 = exhausted). Evaluated against DEFAULT_WRITE_RATE_LIMIT_POLICIES.", |
| 143 | + )}`, |
| 144 | + `# TYPE ${GOVERNOR_RATE_LIMIT_REMAINING_RATIO} gauge`, |
| 145 | + ]; |
| 146 | + for (const row of rateLimitRows) { |
| 147 | + const repoLabel = row.scope === "per_repo" ? `,repo="${escapeLabelValue(row.repoFullName)}"` : ""; |
| 148 | + lines.push( |
| 149 | + `${GOVERNOR_RATE_LIMIT_REMAINING_RATIO}{scope="${row.scope}",action_class="${escapeLabelValue(row.actionClass)}"${repoLabel}} ${row.ratio}`, |
| 150 | + ); |
| 151 | + } |
| 152 | + |
| 153 | + lines.push( |
| 154 | + `# HELP ${GOVERNOR_CAP_USAGE_RATIO} ${escapeMetricsHelpText( |
| 155 | + "The governor's persisted cumulative cap usage as a fraction of DEFAULT_AMS_POLICY_SPEC.capLimits (1 = ceiling reached). dimension is one of budget|turns|elapsed_ms.", |
| 156 | + )}`, |
| 157 | + ); |
| 158 | + lines.push(`# TYPE ${GOVERNOR_CAP_USAGE_RATIO} gauge`); |
| 159 | + for (const row of capRows) { |
| 160 | + lines.push(`${GOVERNOR_CAP_USAGE_RATIO}{dimension="${row.dimension}"} ${row.ratio}`); |
| 161 | + } |
| 162 | + |
| 163 | + return `${lines.join("\n")}\n`; |
| 164 | +} |
| 165 | + |
| 166 | +async function withGovernorState<T>( |
| 167 | + options: GovernorMetricsCliOptions, |
| 168 | + run: (governorState: GovernorState) => T | Promise<T>, |
| 169 | +): Promise<T> { |
| 170 | + const ownsGovernorState = options.openGovernorState === undefined; |
| 171 | + const governorState = (options.openGovernorState ?? openGovernorState)(); |
| 172 | + try { |
| 173 | + return await run(governorState); |
| 174 | + } finally { |
| 175 | + if (ownsGovernorState) governorState.close(); |
| 176 | + } |
| 177 | +} |
| 178 | + |
| 179 | +export async function runGovernorMetrics(args: string[], options: GovernorMetricsCliOptions = {}): Promise<number> { |
| 180 | + if (args.length > 0) { |
| 181 | + return reportCliFailure(argsWantJson(args), GOVERNOR_METRICS_USAGE); |
| 182 | + } |
| 183 | + |
| 184 | + try { |
| 185 | + return await withGovernorState(options, (governorState) => { |
| 186 | + const nowMs = Number.isFinite(options.nowMs) ? (options.nowMs as number) : Date.now(); |
| 187 | + const rateLimitState = governorState.loadRateLimitState(); |
| 188 | + const capUsage = governorState.loadCapUsage(); |
| 189 | + console.log(renderGovernorMetrics(rateLimitState, capUsage, nowMs).trimEnd()); |
| 190 | + return 0; |
| 191 | + }); |
| 192 | + } catch (error) { |
| 193 | + return reportCliFailure(argsWantJson(args), describeCliError(error)); |
| 194 | + } |
| 195 | +} |
0 commit comments