|
2 | 2 | // env, or for one repo, via its .loopover-miner.yml MinerGoalSpec) and records STATE TRANSITIONS to the |
3 | 3 | // append-only governor ledger. Every-check allow/deny recording for a real write action is the fail-closed |
4 | 4 | // Governor chokepoint's job (#2340), which consults this module first in its "safest wins" precedence. |
| 5 | +// |
| 6 | +// PagerDuty paging (#7666): a TRIP transition also fires a page, mirroring ORB's hosted `triggerPagerDutyIncident` |
| 7 | +// (src/services/notify-pagerduty.ts) Events API v2 contract -- same LOOPOVER_ENABLE_PAGERDUTY flag, same |
| 8 | +// PAGERDUTY_ROUTING_KEY, same enqueue URL/payload shape -- with the same simplification #7667's control-plane |
| 9 | +// mirror (control-plane/src/pagerduty-notify.ts) used: no D1/Worker Env here either (the miner is a plain Node |
| 10 | +// process), so no per-repo routing-key map and no severity-threshold/cooldown DB query; PagerDuty's own |
| 11 | +// `dedup_key` still coalesces duplicate incidents. Best-effort: paging can never block or throw past the ledger |
| 12 | +// write it accompanies. |
5 | 13 |
|
6 | 14 | import { |
| 15 | + buildMinerKillSwitchPagerDutyAlert, |
7 | 16 | buildMinerKillSwitchTransitionGovernorLedgerEvent, |
8 | 17 | isGlobalMinerKillSwitch, |
9 | 18 | isMinerKillSwitchActive, |
10 | 19 | resolveMinerKillSwitch, |
11 | 20 | } from "@loopover/engine"; |
12 | | -import type { MinerKillSwitchScope } from "@loopover/engine"; |
| 21 | +import type { MinerKillSwitchPagerDutyAlert, MinerKillSwitchScope } from "@loopover/engine"; |
13 | 22 | import { appendGovernorEvent } from "./governor-ledger.js"; |
14 | 23 | import type { AppendGovernorEventInput, GovernorLedgerEntry } from "./governor-ledger.js"; |
15 | 24 |
|
| 25 | +const PAGERDUTY_EVENTS_URL = "https://events.pagerduty.com/v2/enqueue"; |
| 26 | +// PagerDuty routing/integration keys are 32 lowercase hex characters. |
| 27 | +const ROUTING_KEY_RE = /^[a-f0-9]{32}$/i; |
| 28 | +const TRUTHY_ENV = /^(1|true|yes|on)$/i; |
| 29 | + |
| 30 | +export type NotifyMinerKillSwitchPagerDuty = ( |
| 31 | + alert: MinerKillSwitchPagerDutyAlert, |
| 32 | + env: Record<string, string | undefined>, |
| 33 | +) => void | Promise<void>; |
| 34 | + |
| 35 | +function envString(env: Record<string, string | undefined>, name: string): string | undefined { |
| 36 | + const value = env[name]; |
| 37 | + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; |
| 38 | +} |
| 39 | + |
| 40 | +function warnMinerKillSwitchPagerDutyFailed(dedupKey: string, error: unknown): void { |
| 41 | + const message = (error instanceof Error ? error.message : String(error)).slice(0, 200); |
| 42 | + console.warn(JSON.stringify({ event: "miner_kill_switch_pagerduty_failed", dedupKey, message })); |
| 43 | +} |
| 44 | + |
| 45 | +/** Miner-side mirror of ORB's `triggerPagerDutyIncident` (src/services/notify-pagerduty.ts) Events API v2 |
| 46 | + * contract, same simplification #7667's control-plane mirror used (no D1/Worker Env here either): same |
| 47 | + * LOOPOVER_ENABLE_PAGERDUTY flag, same global PAGERDUTY_ROUTING_KEY, same enqueue URL/payload shape. PagerDuty's |
| 48 | + * own dedup_key still coalesces duplicate incidents. Best-effort: never throws -- a paging failure must never |
| 49 | + * block or mask the governor ledger write it is reporting on. */ |
| 50 | +export async function notifyMinerKillSwitchPagerDuty( |
| 51 | + alert: MinerKillSwitchPagerDutyAlert, |
| 52 | + env: Record<string, string | undefined> = process.env, |
| 53 | +): Promise<void> { |
| 54 | + if (!TRUTHY_ENV.test((env.LOOPOVER_ENABLE_PAGERDUTY ?? "").trim())) return; |
| 55 | + const routingKey = envString(env, "PAGERDUTY_ROUTING_KEY"); |
| 56 | + if (!routingKey || !ROUTING_KEY_RE.test(routingKey)) return; |
| 57 | + |
| 58 | + try { |
| 59 | + const response = await fetch(PAGERDUTY_EVENTS_URL, { |
| 60 | + method: "POST", |
| 61 | + headers: { "content-type": "application/json" }, |
| 62 | + body: JSON.stringify({ |
| 63 | + routing_key: routingKey, |
| 64 | + event_action: "trigger", |
| 65 | + dedup_key: alert.dedupKey, |
| 66 | + payload: { |
| 67 | + summary: alert.summary.slice(0, 1024), |
| 68 | + source: "loopover-miner", |
| 69 | + severity: alert.severity, |
| 70 | + timestamp: new Date().toISOString(), |
| 71 | + component: alert.repoFullName ?? "global", |
| 72 | + custom_details: alert.customDetails, |
| 73 | + }, |
| 74 | + }), |
| 75 | + signal: AbortSignal.timeout(5000), |
| 76 | + }); |
| 77 | + if (!response.ok) { |
| 78 | + console.warn(JSON.stringify({ event: "miner_kill_switch_pagerduty_failed", dedupKey: alert.dedupKey, status: response.status })); |
| 79 | + } |
| 80 | + } catch (error) { |
| 81 | + warnMinerKillSwitchPagerDutyFailed(alert.dedupKey, error); |
| 82 | + } |
| 83 | +} |
| 84 | + |
16 | 85 | export type CheckMinerKillSwitchInput = { |
17 | 86 | repoPaused?: boolean; |
18 | 87 | env?: Record<string, string | undefined>; |
@@ -41,17 +110,46 @@ export type RecordMinerKillSwitchTransitionInput = { |
41 | 110 | scope: MinerKillSwitchScope; |
42 | 111 | }; |
43 | 112 |
|
| 113 | +export type RecordMinerKillSwitchTransitionOptions = { |
| 114 | + append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry; |
| 115 | + /** Injectable for tests; defaults to the real {@link notifyMinerKillSwitchPagerDuty} Events API v2 call. */ |
| 116 | + notify?: NotifyMinerKillSwitchPagerDuty; |
| 117 | + /** Defaults to `process.env`, matching {@link notifyMinerKillSwitchPagerDuty}'s own default. */ |
| 118 | + env?: Record<string, string | undefined>; |
| 119 | +}; |
| 120 | + |
44 | 121 | /** |
45 | 122 | * Record a kill-switch state transition to the governor ledger. No-op (returns null, appends nothing) when the |
46 | 123 | * scope has not actually changed since the previous check — callers own tracking the previous scope (in-memory |
47 | 124 | * or persisted); this module holds no state of its own. |
| 125 | + * |
| 126 | + * On a TRIP (not a resume), also pages PagerDuty (#7666) via {@link notifyMinerKillSwitchPagerDuty}: the ledger |
| 127 | + * row is appended FIRST, then paging is fired fire-and-forget (wrapped in both a sync try/catch and a `.catch` |
| 128 | + * on its returned promise, so neither a synchronous throw nor an async rejection from the notify hook can ever |
| 129 | + * block or mask the ledger write that already landed). |
48 | 130 | */ |
49 | 131 | export function recordMinerKillSwitchTransition( |
50 | 132 | input: RecordMinerKillSwitchTransitionInput, |
51 | | - options: { append?: (event: AppendGovernorEventInput) => GovernorLedgerEntry } = {}, |
| 133 | + options: RecordMinerKillSwitchTransitionOptions = {}, |
52 | 134 | ): GovernorLedgerEntry | null { |
53 | 135 | const event = buildMinerKillSwitchTransitionGovernorLedgerEvent(input); |
54 | 136 | if (!event) return null; |
55 | 137 | const append = options.append ?? appendGovernorEvent; |
56 | | - return append(event as AppendGovernorEventInput); |
| 138 | + const entry = append(event as AppendGovernorEventInput); |
| 139 | + |
| 140 | + const alert = buildMinerKillSwitchPagerDutyAlert(input); |
| 141 | + if (alert) { |
| 142 | + const notify = options.notify ?? notifyMinerKillSwitchPagerDuty; |
| 143 | + const env = options.env ?? process.env; |
| 144 | + try { |
| 145 | + const result = notify(alert, env); |
| 146 | + if (result && typeof (result as Promise<void>).catch === "function") { |
| 147 | + (result as Promise<void>).catch((error: unknown) => warnMinerKillSwitchPagerDutyFailed(alert.dedupKey, error)); |
| 148 | + } |
| 149 | + } catch (error) { |
| 150 | + warnMinerKillSwitchPagerDutyFailed(alert.dedupKey, error); |
| 151 | + } |
| 152 | + } |
| 153 | + |
| 154 | + return entry; |
57 | 155 | } |
0 commit comments