diff --git a/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx b/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx index 248796f64..11830cf89 100644 --- a/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx +++ b/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx @@ -501,6 +501,177 @@ volumes: runner-work-2:`} /> +

Enabling Sentry (your own DSN)

+

+ Sentry is opt-in and off by default. Leave SENTRY_DSN unset + for a complete no-op with negligible overhead — no events leave your box. When you want + error tracking, point the runtime at a project you control in your own Sentry organization. +

+ +

+ Official release images bake GITTENSORY_VERSION as the default release id; + override with SENTRY_RELEASE when you tag custom builds. Mount secrets with{" "} + SENTRY_DSN_FILE instead of inline env when you prefer a file-backed DSN. After + changing Sentry env, restart the gittensory service — there is no hot reload. +

+ + Community self-hosters should send events only to their own DSN. The shipped stack never + phones home to a maintainer-owned project unless you configure one. + + +

Sentry context taxonomy

+

+ Self-host Sentry events carry a small, scrubbed taxonomy so operators can filter by + subsystem without opening raw payloads. Structured error logs forwarded from{" "} + console.error use the JSON event slug as the issue type; direct + captures use a kind or review operation tag instead. +

+ +

+ Cron monitor slugs follow{" "} + gittensory-selfhost-{environment}-{loop} (for example{" "} + gittensory-selfhost-production-scheduled-loop). Pair monitor alerts with queue + depth, dead-job counts, and the matching structured log event. +

+ +

Sentry alert classes and runbook

+

+ Tune Sentry alert rules for persistent failure classes, not one-off + fail-open noise. The table below lists actionable signals, what they usually mean, and the + first checks an operator should run. Prometheus/Grafana alerts in the observability profile + cover the same failure modes from a metrics angle — use both when Sentry is enabled. +

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Alert classSentry signalThreshold guidanceFirst response
Dead-letter growth + selfhost_job_dead, queue_pump_crashed, missed{" "} + queue-dead-letter-revive monitor + + Page when dead jobs stay > 0 for >30m or the revive monitor misses twice + + Check Grafana dead-job panel, /metrics{" "} + gittensory_jobs_dead_total, queue logs; replay from DLQ dashboard only + after fixing root cause +
Check-run permission gaps + check_run_post_denied grouped by repo + + Alert when the same repo hits ≥3 denials in 1h (transient GitHub blips are normal) + + Re-run activation, confirm Checks:write on the GitHub App, verify installation still + has access to the repo +
AI provider exhaustion + close_breaker_engaged, selfhost_ai_provider errors, review + failures tagged operation=ai_review + + Warn on first breaker; page when breaker stays engaged >15m or error rate >25% + of reviews in 1h + + Check provider quotas, AI_* env, CLI availability ( + INSTALL_AI_CLIS), and Grafana AI usage panels +
Relay / broker failures + orb_relay_drain, orb_relay_register,{" "} + orb_broker_unavailable, missed relay monitors + + Page when relay register/drain monitors miss twice or broker errors persist >10m + + Verify Orb enrollment secrets, outbound connectivity, and broker health; check relay + logs without exposing enrollment material +
Backup failures + selfhost_backup_advisory, backup container non-zero exits, stale backup + freshness metric + + Page when backup freshness >2× BACKUP_INTERVAL_SECONDS or verify + script fails twice + + Inspect docker compose logs backup, disk space, and{" "} + backup docs; do not delete the + last good backup after a failed run +
Scheduled monitor misses + Sentry monitor alert on scheduled-loop, orb-export, or + other wrapped loops + + Use Sentry's built-in monitor failure thresholds (2 consecutive misses on most + loops) + + Process may still be alive but cron work stopped — check{" "} + selfhost_cron_error, queue pump logs, and restart the app container if + the loop crashed without taking down the process +
+
+

Sentry server name

SENTRY_SERVER_NAME sets a clean, human name for this instance in Sentry (for @@ -551,9 +722,12 @@ volumes: ]} />

- A missed monitor means the process may still be alive but the recurring work is not checking - in on schedule. Pair the monitor with queue depth, dead-job counts, and the structured error - log for the same subsystem. + Monitor loop slugs (the {loop} segment in the slug) are{" "} + scheduled-loop, orb-export, orb-relay-drain,{" "} + orb-relay-register, and queue-dead-letter-revive. A missed monitor + means the process may still be alive but the recurring work is not checking in on schedule. + Pair the monitor with queue depth, dead-job counts, and the structured error log for the + same subsystem.

Re-gate sweeps (agent-regate-sweep)

diff --git a/src/selfhost/sentry.ts b/src/selfhost/sentry.ts index cf23f0be9..c28a332e3 100644 --- a/src/selfhost/sentry.ts +++ b/src/selfhost/sentry.ts @@ -19,6 +19,9 @@ type SentryNs = typeof import("@sentry/node"); type SentryClient = NonNullable>; type SentryMonitorConfig = NonNullable[1]>; export type SentryMonitorName = "scheduled-loop" | "orb-export" | "orb-relay-drain" | "orb-relay-register" | "queue-dead-letter-revive"; +export const SENTRY_MONITOR_NAMES: readonly SentryMonitorName[] = ["scheduled-loop", "orb-export", "orb-relay-drain", "orb-relay-register", "queue-dead-letter-revive"]; +export const SENTRY_OPERATIONAL_SUBSYSTEMS = { webhook: "GitHub webhook ingest and enqueue", queue: "Job claim, process, dead-letter revival, and pump loops", github: "GitHub App token minting and broker calls", ai: "AI provider attempts, rate limits, and close-breaker engagement", gate: "Gate verdict and check-run publish", publish: "PR comment and public-surface publish", scheduled: "Maintenance tick, regate sweeps, and cron fan-out", backup: "Backup profile runs and freshness advisories", relay: "Orb relay register/drain and broker export loops" } as const; +export const SENTRY_OPERATIONAL_TAG_KEYS = ["repo", "repository", "owner", "installation_id_hash", "pull", "pullNumber", "pr", "head_sha", "project", "kind", "subsystem", "job_type", "jobType", "reason", "result", "deliveryId", "provider", "model", "effort", "timeoutMs", "trace_id", "span_id", "operation", "agent", "decision_outcome", "event", "monitor"] as const; type SentryScope = { setContext(name: string, context: Record): void; setTag(key: string, value: string): void; @@ -34,7 +37,7 @@ let digestHexSync: DigestHex | undefined; const SECRET_KEY = /(token|secret|key|password|passwd|authorization|auth|dsn|cookie|bearer|credential|private)/i; const PAYLOAD_KEY = - /(^|[_-])(body|payload|patch|diff|prompt|rubric|guardrail|headers?|cookies?|title|config|review[-_]?text|review[-_]?content)([_-]|$)|^(body|payload|patch|diff|prompt|rubric|guardrail|headers?|cookies?|title|config|review[-_]?text|review[-_]?content)$/i; + /(^|[_-])(body|payload|patch|diff|prompt|rubric|guardrail|headers?|cookies?|title|config|review[-_]?text|review[-_]?content|comment[-_]?text|comment[-_]?body)([_-]|$)|^(body|payload|patch|diff|prompt|rubric|guardrail|headers?|cookies?|title|config|review[-_]?text|review[-_]?content|comment[-_]?text|comment[-_]?body)$/i; const SECRET_VALUE = new RegExp( [ `${"github" + "_pat_"}[A-Za-z0-9_]+`, @@ -271,6 +274,7 @@ function tagHashedInstallation(scope: SentryScope, context: Record): void { const normalized: Record = typeof context.repository === "string" && context.repo === undefined ? { ...context, repo: context.repository } : { ...context }; tagHashedInstallation(scope, normalized); for (const key of SENTRY_OPERATIONAL_TAG_KEYS) { const tagValue = normalized[key]; if (typeof tagValue === "string" || typeof tagValue === "number") scope.setTag(key, String(tagValue)); } } function scrubString(value: string): string { return value .replace(QUERY_SECRET_VALUE, `$1${REDACTED}`) @@ -435,8 +439,7 @@ export function captureError( if (!active || !Sentry) return; Sentry.withScope((scope) => { setOtelTraceScope(scope); - if (context) scope.setContext("gittensory", hashedInstallationContext(context)); - if (context) tagHashedInstallation(scope, context); + if (context) { const safeContext = hashedInstallationContext(context); scope.setContext("gittensory", safeContext); applyOperationalTags(scope, safeContext); } Sentry!.captureException( error instanceof Error ? error : new Error(String(error)), ); @@ -456,12 +459,7 @@ export function captureReviewFailure( if (context) { const safeContext = hashedInstallationContext(context); scope.setContext("review", safeContext); - tagHashedInstallation(scope, context); - for (const tag of ["owner", "repo", "pr", "head_sha", "operation", "agent", "decision_outcome"]) { - const value = safeContext[tag]; - if (value !== undefined && value !== null) - scope.setTag(tag, String(value)); - } + applyOperationalTags(scope, safeContext); } Sentry!.captureException( error instanceof Error ? error : new Error(String(error)), @@ -469,10 +467,6 @@ export function captureReviewFailure( }); } -// The structured-log fields worth indexing as Sentry tags — the dimensions operators filter + group by. Only -// string|number values are tagged; everything else stays in the full "log" context. -const SENTRY_LOG_TAG_KEYS = ["repo", "repository", "installation_id_hash", "pull", "pullNumber", "pr", "project", "kind", "deliveryId", "provider", "model", "effort", "timeoutMs", "trace_id", "span_id", "operation", "agent", "decision_outcome"] as const; - /** A SHORT location suffix — " (repo#pr)" — for a no-message error title, so the issue list shows WHERE without * dumping every scalar field (which made titles unreadably long, e.g. trailing a full deliveryId). The complete * field set is still indexed as Sentry tags + kept in the "log" context. Empty when the log carries no repo. */ @@ -591,13 +585,8 @@ export function forwardStructuredLogToSentry(line: unknown, fromErrorSink = fals scope.setLevel(severity); setOtelTraceScope(scope); scope.setContext("log", safeObj); - if (event) scope.setTag("event", event); - // Index the dimensions operators filter + group by, so issues are findable without digging into the context. - for (const key of SENTRY_LOG_TAG_KEYS) { - const tagValue = safeObj[key]; - if (typeof tagValue === "string" || typeof tagValue === "number") - scope.setTag(key, String(tagValue)); - } + if (event) safeObj.event = event; + applyOperationalTags(scope, safeObj); // Group recurrences of ONE failure into a single issue (by event, not the variable detail in the value). if (event) scope.setFingerprint(["gittensory-log", event]); // Sentry uses event.transaction as the issue culprit fallback when the stack has no frames; point it at the @@ -642,8 +631,9 @@ export async function withSentryMonitor( Sentry.withScope((scope) => { scope.setLevel("error"); setOtelTraceScope(scope); - scope.setContext("sentry_monitor", safeMonitorContext(name, monitorSlug, context)); - scope.setTag("monitor", monitorSlug); + const monitorContext = safeMonitorContext(name, monitorSlug, context); + scope.setContext("sentry_monitor", monitorContext); + applyOperationalTags(scope, { ...monitorContext, monitor: monitorSlug, kind: `sentry_monitor_${name}`, subsystem: "scheduled" }); scope.setFingerprint(["gittensory-sentry-monitor", name]); Sentry!.captureException(error instanceof Error ? error : new Error(String(error))); }); diff --git a/test/unit/docs-selfhost-sentry-observability.test.ts b/test/unit/docs-selfhost-sentry-observability.test.ts new file mode 100644 index 000000000..cf75379b0 --- /dev/null +++ b/test/unit/docs-selfhost-sentry-observability.test.ts @@ -0,0 +1,48 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +import { + SENTRY_MONITOR_NAMES, + SENTRY_OPERATIONAL_SUBSYSTEMS, + SENTRY_OPERATIONAL_TAG_KEYS, +} from "../../src/selfhost/sentry"; + +// Drift guard (#1824): self-host Sentry docs must stay aligned with exported taxonomy and runbook signals. + +const OPERATIONS = "apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx"; +const operations = readFileSync(OPERATIONS, "utf8"); + +describe("self-host Sentry observability docs (#1824)", () => { + it("documents enabling Sentry with an operator-owned DSN", () => { + expect(operations).toContain("Enabling Sentry (your own DSN)"); + expect(operations).toContain("SENTRY_DSN"); + expect(operations).toContain("opt-in and off by default"); + expect(operations).toContain("SENTRY_DSN_FILE"); + }); + + it("documents context taxonomy and cron monitor slugs", () => { + expect(operations).toContain("Sentry context taxonomy"); + expect(operations).toContain("installation_id_hash"); + for (const monitor of SENTRY_MONITOR_NAMES) { + expect(operations).toContain(monitor); + } + for (const subsystem of Object.keys(SENTRY_OPERATIONAL_SUBSYSTEMS)) { + expect(operations).toContain(subsystem); + } + for (const tag of ["kind", "subsystem", "jobType", "operation"]) { + expect(SENTRY_OPERATIONAL_TAG_KEYS).toContain(tag); + expect(operations).toContain(tag); + } + expect(operations).toContain("repository"); + }); + + it("documents alert classes and runbook first-response checks", () => { + expect(operations).toContain("Sentry alert classes and runbook"); + expect(operations).toContain("selfhost_job_dead"); + expect(operations).toContain("check_run_post_denied"); + expect(operations).toContain("close_breaker_engaged"); + expect(operations).toContain("orb_broker_unavailable"); + expect(operations).toContain("selfhost_backup_advisory"); + expect(operations).toContain("scheduled-loop"); + }); +}); diff --git a/test/unit/selfhost-sentry.test.ts b/test/unit/selfhost-sentry.test.ts index 2c1716fec..b7f5a15e4 100644 --- a/test/unit/selfhost-sentry.test.ts +++ b/test/unit/selfhost-sentry.test.ts @@ -75,6 +75,9 @@ import { scrubEvent, resetSentryForTest, withSentryMonitor, + SENTRY_MONITOR_NAMES, + SENTRY_OPERATIONAL_SUBSYSTEMS, + SENTRY_OPERATIONAL_TAG_KEYS, } from "../../src/selfhost/sentry"; beforeEach(() => { @@ -197,6 +200,32 @@ describe("scrubEvent — redact secrets before an event leaves the box", () => { expect(ev.tags.authToken).toBe("[redacted]"); }); + it("preserves contexts.review repo/pr while redacting review text fields (#1824, PR #1881 regression)", () => { + const ev = scrubbedEvent({ + contexts: { + review: { + repo: "owner/repo", + pr: 7, + head_sha: "abc123", + operation: "gate_decision", + reviewText: "private review body", + reviewBody: "also private", + commentBody: "private comment", + comment_text: "private comment text", + }, + }, + }) as any; + + expect(ev.contexts.review.repo).toBe("owner/repo"); + expect(ev.contexts.review.pr).toBe(7); + expect(ev.contexts.review.head_sha).toBe("abc123"); + expect(ev.contexts.review.operation).toBe("gate_decision"); + expect(ev.contexts.review.reviewText).toBe("[redacted]"); + expect(ev.contexts.review.reviewBody).toBe("[redacted]"); + expect(ev.contexts.review.commentBody).toBe("[redacted]"); + expect(ev.contexts.review.comment_text).toBe("[redacted]"); + }); + it("scrubs request URL/query fields and deletes top-level user data", () => { const queryTokenKey = fakeQueryTokenKey(); const ev = scrubbedEvent({ @@ -493,14 +522,16 @@ describe("enabled when SENTRY_DSN is set", () => { ).toBe("custom@sha"); }); - it("captureError sends with context, and without context skips setContext", async () => { + it("captureError sends with context, tags operational fields, and without context skips setContext", async () => { await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); captureError(new Error("boom"), { kind: "job_dead" }); expect(mocks.scope.setContext).toHaveBeenCalledWith("gittensory", { kind: "job_dead", }); + expect(mocks.scope.setTag).toHaveBeenCalledWith("kind", "job_dead"); expect(mocks.captureException).toHaveBeenCalledTimes(1); mocks.scope.setContext.mockClear(); + mocks.scope.setTag.mockClear(); captureError(new Error("invalid install"), { kind: "job_dead", installation_id: "not-an-installation", @@ -508,6 +539,7 @@ describe("enabled when SENTRY_DSN is set", () => { expect(mocks.scope.setContext).toHaveBeenCalledWith("gittensory", { kind: "job_dead", }); + expect(mocks.scope.setTag).toHaveBeenCalledWith("kind", "job_dead"); mocks.scope.setContext.mockClear(); captureError("plain string with no context"); expect(mocks.scope.setContext).not.toHaveBeenCalled(); @@ -762,6 +794,12 @@ describe("enabled when SENTRY_DSN is set", () => { "monitor", "gittensory-selfhost-prod-orb-export", ); + expect(mocks.scope.setTag).toHaveBeenCalledWith("jobType", "orb-export"); + expect(mocks.scope.setTag).toHaveBeenCalledWith( + "kind", + "sentry_monitor_orb-export", + ); + expect(mocks.scope.setTag).toHaveBeenCalledWith("subsystem", "scheduled"); expect(mocks.scope.setFingerprint).toHaveBeenCalledWith([ "gittensory-sentry-monitor", "orb-export", @@ -926,6 +964,21 @@ describe("forwardStructuredLogToSentry — central console.log → Sentry error expect(mocks.scope.setTag).toHaveBeenCalledWith("timeoutMs", "240000"); }); + it("normalizes repository log alias to repo tag (#1881 nit)", async () => { + await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); + forwardStructuredLogToSentry( + JSON.stringify({ + level: "error", + event: "check_run_post_denied", + repository: "owner/repo", + pullNumber: 3, + message: "permission denied", + }), + ); + expect(mocks.scope.setTag).toHaveBeenCalledWith("repo", "owner/repo"); + expect(mocks.scope.setTag).toHaveBeenCalledWith("repository", "owner/repo"); + }); + it("indexes trace ids already present on structured error logs", async () => { await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); forwardStructuredLogToSentry( @@ -1151,3 +1204,46 @@ describe("installStructuredLogForwarding — central console sink instrumentatio expect(base.error).toHaveBeenCalledTimes(2); }); }); + +describe("Sentry operational taxonomy exports (#1824)", () => { + it("exports monitor names aligned with wrapped cron loops", () => { + expect(SENTRY_MONITOR_NAMES).toEqual([ + "scheduled-loop", + "orb-export", + "orb-relay-drain", + "orb-relay-register", + "queue-dead-letter-revive", + ]); + }); + + it("exports subsystem taxonomy keys for operator docs", () => { + expect(Object.keys(SENTRY_OPERATIONAL_SUBSYSTEMS).sort()).toEqual([ + "ai", + "backup", + "gate", + "github", + "publish", + "queue", + "relay", + "scheduled", + "webhook", + ]); + }); + + it("keeps operational tag keys low-cardinality and review-aware", () => { + expect(SENTRY_OPERATIONAL_TAG_KEYS).toEqual( + expect.arrayContaining([ + "kind", + "subsystem", + "jobType", + "repo", + "pr", + "head_sha", + "operation", + "event", + "monitor", + ]), + ); + expect(SENTRY_OPERATIONAL_TAG_KEYS).not.toContain("installationId"); + }); +});