Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .loopover.yml.example
Original file line number Diff line number Diff line change
Expand Up @@ -1443,6 +1443,13 @@ settings:
# activeReviewReconciliation:
# enabled: true # Bool. Default: false (the env var decides instead).

# Fleet-wide Rent-a-Loop escalation sweep (#6349 / #8018): config-as-code override for the
# LOOPOVER_LOOP_ESCALATION flag that gates the hourly fleet-escalation notification. Same shape and
# precedence as `ops:` / `activeReviewReconciliation:` above — operator-level, only meaningful on the
# loopover self-repo's own manifest.
# loopEscalation:
# enabled: true # Bool. Default: false (the env var decides instead).

# Opt-in federated fleet intelligence export (#1970): packages this instance's OWN calibration signals into a
# signed, anonymized bundle an operator can choose to hand to a peer (or to a collector they run). Exports
# AGGREGATE figures only -- gate precision, reversal/slop/copycat rates over a window -- never source code,
Expand Down
7 changes: 7 additions & 0 deletions config/examples/loopover.full.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1457,6 +1457,13 @@ settings:
# activeReviewReconciliation:
# enabled: true # Bool. Default: false (the env var decides instead).

# Fleet-wide Rent-a-Loop escalation sweep (#6349 / #8018): config-as-code override for the
# LOOPOVER_LOOP_ESCALATION flag that gates the hourly fleet-escalation notification. Same shape and
# precedence as `ops:` / `activeReviewReconciliation:` above — operator-level, only meaningful on the
# loopover self-repo's own manifest.
# loopEscalation:
# enabled: true # Bool. Default: false (the env var decides instead).

# Opt-in federated fleet intelligence export (#1970): packages this instance's OWN calibration signals into a
# signed, anonymized bundle an operator can choose to hand to a peer (or to a collector they run). Exports
# AGGREGATE figures only -- gate precision, reversal/slop/copycat rates over a window -- never source code,
Expand Down
1 change: 1 addition & 0 deletions packages/loopover-engine/src/config-lint.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const TOP_LEVEL_FIELDS = [
"sweepWatchdog",
"prReconciliation",
"activeReviewReconciliation",
"loopEscalation",
"federatedIntelligence",
] as const;

Expand Down
3 changes: 3 additions & 0 deletions packages/loopover-engine/src/focus-manifest-validation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
sweepWatchdogConfigToJson,
prReconciliationConfigToJson,
activeReviewReconciliationConfigToJson,
loopEscalationConfigToJson,
federatedIntelligenceConfigToJson,
settingsOverrideToJson,
type FocusManifest,
Expand Down Expand Up @@ -98,6 +99,8 @@ function focusManifestToNormalizedJson(manifest: FocusManifest): Record<string,
if (prReconciliation !== null) normalized.prReconciliation = prReconciliation;
const activeReviewReconciliation = activeReviewReconciliationConfigToJson(manifest.activeReviewReconciliation);
if (activeReviewReconciliation !== null) normalized.activeReviewReconciliation = activeReviewReconciliation;
const loopEscalation = loopEscalationConfigToJson(manifest.loopEscalation);
if (loopEscalation !== null) normalized.loopEscalation = loopEscalation;
const federatedIntelligence = federatedIntelligenceConfigToJson(manifest.federatedIntelligence);
if (federatedIntelligence !== null) normalized.federatedIntelligence = federatedIntelligence;

Expand Down
44 changes: 44 additions & 0 deletions packages/loopover-engine/src/focus-manifest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -508,6 +508,18 @@ export type FocusManifestActiveReviewReconciliationConfig = {
enabled: boolean;
};

/**
* Config-as-code override for the Rent-a-Loop fleet-escalation sweep (LOOPOVER_LOOP_ESCALATION), declared
* under top-level `loopEscalation:` (#8018). Same shape and precedence as `ops:` / `activeReviewReconciliation:`
* above -- fleet-wide, self-repo-manifest-sourced, no DB-backed counterpart. Not present ⇒ the caller falls
* back to the LOOPOVER_LOOP_ESCALATION env var. Distinct from the four review-evasion / reconciliation
* sweeps: this one pages an operator when rented loops need attention.
*/
export type FocusManifestLoopEscalationConfig = {
present: boolean;
enabled: boolean;
};

/**
* Config-as-code opt-in for the federated fleet intelligence export (#1970), declared under
* `federatedIntelligence:`. Gates buildFederatedBundle (src/orb/federated-bundle.ts), which packages this
Expand Down Expand Up @@ -1217,6 +1229,7 @@ export type FocusManifest = {
sweepWatchdog: FocusManifestSweepWatchdogConfig;
prReconciliation: FocusManifestPrReconciliationConfig;
activeReviewReconciliation: FocusManifestActiveReviewReconciliationConfig;
loopEscalation: FocusManifestLoopEscalationConfig;
federatedIntelligence: FocusManifestFederatedIntelligenceConfig;
warnings: string[];
};
Expand Down Expand Up @@ -1402,6 +1415,11 @@ const EMPTY_ACTIVE_REVIEW_RECONCILIATION_CONFIG: FocusManifestActiveReviewReconc
enabled: false,
};

const EMPTY_LOOP_ESCALATION_CONFIG: FocusManifestLoopEscalationConfig = {
present: false,
enabled: false,
};

const EMPTY_FEDERATED_INTELLIGENCE_CONFIG: FocusManifestFederatedIntelligenceConfig = {
present: false,
enabled: false,
Expand Down Expand Up @@ -1437,6 +1455,7 @@ const EMPTY_MANIFEST: FocusManifest = {
sweepWatchdog: { ...EMPTY_SWEEP_WATCHDOG_CONFIG },
prReconciliation: { ...EMPTY_PR_RECONCILIATION_CONFIG },
activeReviewReconciliation: { ...EMPTY_ACTIVE_REVIEW_RECONCILIATION_CONFIG },
loopEscalation: { ...EMPTY_LOOP_ESCALATION_CONFIG },
federatedIntelligence: { ...EMPTY_FEDERATED_INTELLIGENCE_CONFIG },
warnings: [],
};
Expand Down Expand Up @@ -1478,6 +1497,7 @@ function emptyManifest(source: FocusManifestSource, warnings: string[] = []): Fo
sweepWatchdog: { ...EMPTY_SWEEP_WATCHDOG_CONFIG },
prReconciliation: { ...EMPTY_PR_RECONCILIATION_CONFIG },
activeReviewReconciliation: { ...EMPTY_ACTIVE_REVIEW_RECONCILIATION_CONFIG },
loopEscalation: { ...EMPTY_LOOP_ESCALATION_CONFIG },
federatedIntelligence: { ...EMPTY_FEDERATED_INTELLIGENCE_CONFIG },
};
}
Expand Down Expand Up @@ -2381,6 +2401,28 @@ export function activeReviewReconciliationConfigToJson(config: FocusManifestActi
return { enabled: config.enabled };
}

/**
* Parse the optional top-level `loopEscalation:` mapping (#8018). Mirrors {@link parseOpsConfig} exactly —
* `enabled` is the only field.
*/
function parseLoopEscalationConfig(value: JsonValue | undefined, warnings: string[]): FocusManifestLoopEscalationConfig {
if (value === undefined || value === null) return { ...EMPTY_LOOP_ESCALATION_CONFIG };
if (typeof value !== "object" || Array.isArray(value)) {
warnings.push('Manifest field "loopEscalation" must be a mapping; ignoring it.');
return { ...EMPTY_LOOP_ESCALATION_CONFIG };
}
const record = value as Record<string, JsonValue>;
const enabled = normalizeOptionalBoolean(record.enabled, "loopEscalation.enabled", warnings) ?? false;
return { present: true, enabled };
}

/** Serialize a loopEscalation config back into the parse-compatible shape so a cached snapshot round-trips
* through {@link parseLoopEscalationConfig} unchanged. Returns null when nothing is configured. */
export function loopEscalationConfigToJson(config: FocusManifestLoopEscalationConfig): JsonValue {
if (!config.present) return null;
return { enabled: config.enabled };
}

/**
* Parse the optional `federatedIntelligence:` mapping (#1970). Mirrors {@link parseUpstreamDriftIssuesConfig}
* exactly -- `enabled` is the only field, defaulting to false, so the parsed value IS the effective value and
Expand Down Expand Up @@ -3944,6 +3986,7 @@ export function parseFocusManifest(raw: unknown, source?: FocusManifestSource):
sweepWatchdog: parseSweepWatchdogConfig(record.sweepWatchdog, warnings),
prReconciliation: parsePrReconciliationConfig(record.prReconciliation, warnings),
activeReviewReconciliation: parseActiveReviewReconciliationConfig(record.activeReviewReconciliation, warnings),
loopEscalation: parseLoopEscalationConfig(record.loopEscalation, warnings),
federatedIntelligence: parseFederatedIntelligenceConfig(record.federatedIntelligence, warnings),
warnings,
};
Expand Down Expand Up @@ -3971,6 +4014,7 @@ export function parseFocusManifest(raw: unknown, source?: FocusManifestSource):
!manifest.sweepWatchdog.present &&
!manifest.prReconciliation.present &&
!manifest.activeReviewReconciliation.present &&
!manifest.loopEscalation.present &&
!manifest.federatedIntelligence.present
) {
warnings.push("Manifest contained no recognized focus fields; falling back to deterministic signals.");
Expand Down
3 changes: 3 additions & 0 deletions packages/loopover-engine/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -790,6 +790,7 @@ export {
sweepWatchdogConfigToJson,
prReconciliationConfigToJson,
activeReviewReconciliationConfigToJson,
loopEscalationConfigToJson,
federatedIntelligenceConfigToJson,
FEDERATED_COLLECTOR_MODES,
settingsOverrideToJson,
Expand Down Expand Up @@ -832,6 +833,8 @@ export {
type FocusManifestUpstreamDriftIssuesConfig,
type FocusManifestSweepWatchdogConfig,
type FocusManifestPrReconciliationConfig,
type FocusManifestActiveReviewReconciliationConfig,
type FocusManifestLoopEscalationConfig,
type FocusManifestFederatedIntelligenceConfig,
type FederatedCollectorMode,
type FocusManifestSettings,
Expand Down
16 changes: 12 additions & 4 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { gittensorEnabledRepoFullNames } from "./review/gittensor-wire";
import { isOpsEnabled, resolveOpsManifestOverride } from "./review/ops-wire";
import { isRecapEnabled, resolveMaintainerRecapManifestOverride, shouldFireMaintainerRecap } from "./review/maintainer-recap-wire";
import { isSweepWatchdogEnabled, resolveSweepWatchdogManifestOverride } from "./review/sweep-watchdog";
import { isLoopEscalationSweepEnabled } from "./review/loop-escalation-wire";
import { isLoopEscalationSweepEnabled, resolveLoopEscalationManifestOverride } from "./review/loop-escalation-wire";
import { isPrReconciliationEnabled, resolvePrReconciliationManifestOverride } from "./review/pr-reconciliation";
import { isActiveReviewReconciliationEnabled, resolveActiveReviewReconciliationManifestOverride } from "./review/active-review-reconciliation";
import { isRagEnabled } from "./review/rag-wire";
Expand Down Expand Up @@ -264,9 +264,17 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController):
if (isSweepWatchdogEnabled(env, sweepWatchdogManifestOverride)) jobs.push({ type: "sweep-liveness-watchdog", requestedBy: "schedule" });
}
// Rent-a-Loop escalation (#6349, flag LOOPOVER_LOOP_ESCALATION). Hourly fleet summary → Discord when
// needingAttention is non-empty. Enqueued ONLY when the flag is ON — flag-OFF (default) this job is never
// created, so the cron tick does ZERO new work and the enqueued set is byte-identical to today.
if (selfHostedReviews && isLoopEscalationSweepEnabled(env)) jobs.push({ type: "loop-escalation-sweep", requestedBy: "schedule" });
// needingAttention is non-empty. Enable can ALSO be set as code via the loopover self-repo's
// `.loopover.yml loopEscalation:` block (config-as-code parity, #8018) -- a present manifest block wins
// over the env var; absent, the env var decides exactly as before. Enqueued ONLY when enabled — flag-OFF
// (default) this job is never created, so the cron tick does ZERO new work and the enqueued set is
// byte-identical to today.
if (selfHostedReviews) {
const loopEscalationManifestOverride = await resolveLoopEscalationManifestOverride(env);
if (isLoopEscalationSweepEnabled(env, loopEscalationManifestOverride)) {
jobs.push({ type: "loop-escalation-sweep", requestedBy: "schedule" });
}
}
// Convergence (self-improve / auto-tune, flag LOOPOVER_REVIEW_SELFTUNE). Hourly self-improvement tick over
// loopover's own review-outcome data: compute tuning recommendations, shadow-soak any strictly-tightening
// one, and auto-promote it to live only after the soak window passes the gate (TIGHTENING-ONLY, audited).
Expand Down
14 changes: 9 additions & 5 deletions src/queue/job-dispatch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import { executeAgentRun } from "../services/agent-orchestrator";
import { deliverNotification, evaluateNotificationEvent } from "../notifications/service";
import { isOpsEnabled, resolveOpsManifestOverride, runOpsAlerts } from "../review/ops-wire";
import { isSweepWatchdogEnabled, resolveSweepWatchdogManifestOverride, runSweepLivenessWatchdog } from "../review/sweep-watchdog";
import { isLoopEscalationSweepEnabled, runLoopEscalationSweep } from "../review/loop-escalation-wire";
import { isLoopEscalationSweepEnabled, resolveLoopEscalationManifestOverride, runLoopEscalationSweep } from "../review/loop-escalation-wire";
import { isPrReconciliationEnabled, resolvePrReconciliationManifestOverride, runOpenPrReconciliation } from "../review/pr-reconciliation";
import { isActiveReviewReconciliationEnabled, resolveActiveReviewReconciliationManifestOverride, runActiveReviewReconciliation } from "../review/active-review-reconciliation";
import { isSelfTuneEnabled, runSelfTune } from "../review/selftune-wire";
Expand Down Expand Up @@ -308,10 +308,14 @@ export async function processJob(env: Env, message: JobMessage): Promise<void> {
}
return;
case "loop-escalation-sweep":
// Rent-a-Loop escalation (#6349, flag LOOPOVER_LOOP_ESCALATION). Defense-in-depth: the cron only
// ENQUEUES this when the flag is ON, but a stale in-flight job that lands after a flag-flip must still
// no-op. Fails safe internally — never throws into the queue.
if (isLoopEscalationSweepEnabled(env)) await runLoopEscalationSweep(env);
// Rent-a-Loop escalation (#6349, flag LOOPOVER_LOOP_ESCALATION, config-as-code override #8018). Defense-
// in-depth: the cron only ENQUEUES this when enabled, but a stale in-flight job that lands after a
// flag-flip (env OR manifest) must still no-op, so disabled does zero work here too. Fails safe
// internally — never throws into the queue.
{
const loopEscalationManifestOverride = await resolveLoopEscalationManifestOverride(env);
if (isLoopEscalationSweepEnabled(env, loopEscalationManifestOverride)) await runLoopEscalationSweep(env);
}
return;
case "reconcile-open-prs":
// Self-heal (flag LOOPOVER_PR_RECONCILIATION). Defense-in-depth: the cron only ENQUEUES this when
Expand Down
56 changes: 53 additions & 3 deletions src/review/loop-escalation-wire.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@
// open condition does not re-page every tick.
//
// Default OFF (LOOPOVER_LOOP_ESCALATION) — flag-OFF the cron enqueues no job and this module is never invoked,
// byte-identical to today. There is no rented-loop D1 store yet; the loader reads LOOPOVER_ACTIVE_LOOPS_JSON
// byte-identical to today. Enable can ALSO be set as code via the loopover self-repo's `.loopover.yml
// loopEscalation:` block (config-as-code parity, #8018) — a present manifest block wins over the env var.
// There is no rented-loop D1 store yet; the loader reads LOOPOVER_ACTIVE_LOOPS_JSON
// (a JSON array of ActiveLoopFacts) so a simulated escalation-worthy loop can reach a human without waiting on
// the separate observability-store work (#4793). Callers may inject `loadActiveLoops` in tests.
/* v8 ignore file -- thoroughly unit-tested in test/unit/loop-escalation-wire.test.ts; codecov patch still
Expand All @@ -21,18 +23,66 @@ import {
type FleetLoopRow,
} from "../../packages/loopover-engine/src/loop-fleet-summary";
import { countRecentAuditEventsForActorAndTarget, recordAuditEvent } from "../db/repositories";
import { loadRepoFocusManifest } from "../signals/focus-manifest-loader";
import { resolveLoopOverSelfRepoFullName } from "../config/loopover-repo-focus-manifest";
import { errorMessage } from "../utils/json";

const ALLOWED_DISCORD_HOSTS = new Set(["discord.com", "discordapp.com"]);
const DEFAULT_COOLDOWN_MINUTES = 60;
const AUDIT_EVENT_TYPE = "loop_escalation_notification.discord";
const AUDIT_TARGET_KEY = "fleet:loop-escalation";

/** True when the scheduled fleet-escalation sweep is enabled. Default OFF. */
export function isLoopEscalationSweepEnabled(env: { LOOPOVER_LOOP_ESCALATION?: string | undefined }): boolean {
/** A manifest-sourced enable override (#8018) -- the top-level `loopEscalation` block of the loopover
* self-repo's `.loopover.yml` (see FocusManifestLoopEscalationConfig). `present: false` (no block, or the
* repo has no manifest at all) means "no override configured", not "disabled" -- the caller falls through
* to the env var in that case, exactly as if this parameter were omitted. Mirrors OpsManifestOverride. */
export type LoopEscalationManifestOverride = { present: boolean; enabled: boolean };

/** True when the scheduled fleet-escalation sweep is enabled. Config-as-code (#8018): a present top-level
* `loopEscalation` manifest block on the loopover self-repo wins outright; otherwise falls back to the
* LOOPOVER_LOOP_ESCALATION env flag (default OFF). */
export function isLoopEscalationSweepEnabled(
env: { LOOPOVER_LOOP_ESCALATION?: string | undefined },
manifestOverride?: LoopEscalationManifestOverride | undefined,
): boolean {
if (manifestOverride?.present) return manifestOverride.enabled;
return /^(1|true|yes|on)$/i.test((env.LOOPOVER_LOOP_ESCALATION ?? "").trim());
}

// Short in-isolate TTL cache for resolveLoopEscalationManifestOverride, mirroring ops-wire.ts: the override
// always resolves to the SAME repo (resolveLoopOverSelfRepoFullName is fleet-wide), so a single slot
// suffices. Called from the scheduled cron tick AND the queue's loop-escalation-sweep job.
const LOOP_ESCALATION_MANIFEST_OVERRIDE_CACHE_TTL_MS = 60_000;
let loopEscalationManifestOverrideCache: { override: LoopEscalationManifestOverride; at: number } | null = null;

/**
* Config-as-code override lookup (#8018): read the top-level `loopEscalation` block off the loopover
* self-repo's `.loopover.yml`. A manifest load failure degrades to `{ present: false }` so a hiccup can
* never accidentally enable or disable the sweep. `nowMs` defaults to `Date.now()` so callers need no
* change, while tests can pass a deterministic value to exercise the TTL precisely.
*/
export async function resolveLoopEscalationManifestOverride(env: Env, nowMs: number = Date.now()): Promise<LoopEscalationManifestOverride> {
const hit = loopEscalationManifestOverrideCache;
if (hit && nowMs - hit.at < LOOP_ESCALATION_MANIFEST_OVERRIDE_CACHE_TTL_MS) return hit.override;
try {
const manifest = await loadRepoFocusManifest(env, resolveLoopOverSelfRepoFullName(env));
const config = manifest.loopEscalation;
const override = { present: config.present, enabled: config.enabled };
loopEscalationManifestOverrideCache = { override, at: nowMs };
return override;
} catch (error) {
console.warn(JSON.stringify({ event: "loop_escalation_manifest_override_error", message: errorMessage(error).slice(0, 200) }));
const override = { present: false, enabled: false };
loopEscalationManifestOverrideCache = { override, at: nowMs };
return override;
}
}

/** Test-only: clears the cached override, mirroring clearOpsManifestOverrideCacheForTest. */
export function clearLoopEscalationManifestOverrideCacheForTest(): void {
loopEscalationManifestOverrideCache = null;
}

function envString(env: Env, name: string): string | undefined {
const fromEnv = (env as unknown as Record<string, unknown>)[name];
return typeof fromEnv === "string" && fromEnv.trim().length > 0 ? fromEnv.trim() : undefined;
Expand Down
Loading