diff --git a/review-enrichment/README.md b/review-enrichment/README.md index 7859bb96c..1ed829245 100644 --- a/review-enrichment/README.md +++ b/review-enrichment/README.md @@ -159,6 +159,33 @@ elapsed time, partial/analyzer status, history lookup counts, GitHub endpoint ca those fields to spot a broken analyzer without exposing request bodies, diffs, tokens, prompts, comments, or private config. +### REES Sentry queries + +REES keeps its indexed Sentry tags intentionally small and stable: + +- `event` +- `route` +- `method` +- `repo` +- `pullNumber` +- `analyzer` +- `release` +- `environment` +- `railwayDeploymentId` + +Useful production queries: + +- Route exceptions on the enrichment endpoint: + - `event:rees_route_error route:/v1/enrich method:POST` +- Analyzer failures grouped by analyzer: + - `event:rees_analyzer_degraded analyzer:history` + - `event:rees_analyzer_degraded analyzer:dependency repo:JSONbored/gittensory` +- Source-map upload/startup failures on a Railway deploy: + - `event:rees_sourcemap_upload_failed railwayDeploymentId:` +- Process-level crashes: + - `event:rees_uncaught_exception` + - `event:rees_unhandled_rejection` + If Sentry still shows frames such as `/app/dist/server.js`, check: 1. The event's `release` is `gittensory-rees@` or your exact `SENTRY_RELEASE` override. diff --git a/review-enrichment/src/sentry.ts b/review-enrichment/src/sentry.ts index c178dbbf6..4904ea065 100644 --- a/review-enrichment/src/sentry.ts +++ b/review-enrichment/src/sentry.ts @@ -2,6 +2,12 @@ import type { ErrorEvent, EventHint } from "@sentry/node"; type SentryNs = typeof import("@sentry/node"); type SentryClient = Pick; +type SentryScope = { + setContext(name: string, context: Record): unknown; + setFingerprint(fingerprint: string[]): unknown; + setLevel(level: "error" | "warning"): unknown; + setTag(key: string, value: string): unknown; +}; let Sentry: SentryClient | undefined; let active = false; @@ -10,6 +16,27 @@ let activeEnvironment = "production"; const SECRET_FIELD = /(?:authorization|cookie|token|secret|password|private[_-]?key|shared[_-]?secret)/i; const SECRET_VALUE = /\b(?:github_pat_[A-Za-z0-9_]+|gh[pousr]_[A-Za-z0-9_]+|gts_[a-f0-9]{64}|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)\b/g; +const REES_SENTRY_TAG_KEYS = [ + "event", + "route", + "method", + "repo", + "pullNumber", + "analyzer", + "release", + "environment", + "railwayDeploymentId", +] as const; + +type ReesSentryTagKey = (typeof REES_SENTRY_TAG_KEYS)[number]; +type ReesSentryTags = Partial>; +type ReesCaptureOptions = { + contextName: string; + context: Record; + fingerprint: string[]; + level?: "error" | "warning"; + tags: ReesSentryTags; +}; function nonBlank(value: string | undefined): string | undefined { const text = value?.trim(); @@ -65,6 +92,35 @@ function compactContext(value: Record): Record return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)); } +function setAllowedTags(scope: Pick, tags: ReesSentryTags): void { + for (const key of REES_SENTRY_TAG_KEYS) { + const value = sentryTagValue(tags[key]); + if (value) scope.setTag(key, value); + } +} + +function setFingerprint(scope: Pick, parts: string[]): void { + const safeParts = parts.map((part) => sentryTagValue(part) ?? "unknown"); + scope.setFingerprint(safeParts); +} + +function captureScopedError(error: unknown, options: ReesCaptureOptions): void { + if (!active || !Sentry) return; + const safeContext = scrubValue(compactContext(options.context)) as Record; + Sentry.withScope((scope) => { + scope.setLevel(options.level ?? "error"); + scope.setContext(options.contextName, safeContext); + setFingerprint(scope, options.fingerprint); + setAllowedTags(scope, { + ...options.tags, + event: options.tags.event, + release: options.tags.release ?? activeRelease, + environment: options.tags.environment ?? activeEnvironment, + }); + Sentry!.captureException(error instanceof Error ? error : new Error(String(error))); + }); +} + function scrubEvent(event: ErrorEvent): ErrorEvent { return scrubValue(event) as ErrorEvent; } @@ -95,10 +151,85 @@ export async function initSentry(env: NodeJS.ProcessEnv): Promise { } export function captureError(error: unknown, context?: Record): void { - if (!active || !Sentry) return; - Sentry.withScope((scope) => { - if (context) scope.setContext("rees", scrubValue(context) as Record); - Sentry!.captureException(error instanceof Error ? error : new Error(String(error))); + captureScopedError(error, { + contextName: "rees", + context: { + ...context, + release: activeRelease, + environment: activeEnvironment, + }, + fingerprint: ["rees-error"], + tags: {}, + }); +} + +export function captureRouteError( + error: unknown, + context: { route: string; method: string }, +): void { + captureScopedError(error, { + contextName: "rees_route", + context: { + event: "rees_route_error", + route: context.route, + method: context.method, + release: activeRelease, + environment: activeEnvironment, + }, + fingerprint: ["rees-route-error", context.route, context.method], + tags: { + event: "rees_route_error", + route: context.route, + method: context.method, + }, + }); +} + +export function captureUnhandledError( + error: unknown, + context: { event: "rees_unhandled_rejection" | "rees_uncaught_exception" }, +): void { + captureScopedError(error, { + contextName: "rees_process", + context: { + event: context.event, + release: activeRelease, + environment: activeEnvironment, + }, + fingerprint: ["rees-process-error", context.event], + tags: { + event: context.event, + }, + }); +} + +export function captureSourcemapUploadFailure( + error: unknown, + context: { + release?: string; + railwayDeploymentId?: string; + strict?: boolean; + sha?: string; + stage?: string; + }, +): void { + captureScopedError(error, { + contextName: "rees_sourcemap_upload", + context: { + event: "rees_sourcemap_upload_failed", + release: context.release ?? activeRelease, + railwayDeploymentId: context.railwayDeploymentId, + strict: context.strict, + sha: context.sha, + stage: context.stage, + environment: activeEnvironment, + }, + fingerprint: ["rees-sourcemap-upload-failed"], + tags: { + event: "rees_sourcemap_upload_failed", + release: context.release ?? activeRelease, + railwayDeploymentId: context.railwayDeploymentId, + }, }); } @@ -138,9 +269,10 @@ export interface AnalyzerDegradationContext { } export function captureAnalyzerDegradation(error: unknown, context: AnalyzerDegradationContext): void { - if (!active || !Sentry) return; const headShaPrefix = nonBlank(context.headSha)?.slice(0, 12); - const safeContext = { + captureScopedError(error, { + contextName: "rees_analyzer", + context: { event: "rees_analyzer_degraded", analyzer: context.analyzer, requestedAnalyzers: context.requestedAnalyzers, @@ -176,50 +308,14 @@ export function captureAnalyzerDegradation(error: unknown, context: AnalyzerDegr traceId: context.traceId, release: activeRelease, environment: activeEnvironment, - }; - Sentry.withScope((scope) => { - const analyzerTag = sentryTagValue(context.analyzer) ?? "unknown"; - const headShaTag = sentryTagValue(headShaPrefix); - const timeoutTag = sentryTagValue(context.timeoutMs); - const releaseTag = sentryTagValue(activeRelease); - scope.setLevel("error"); - scope.setContext("rees_analyzer", scrubValue(compactContext(safeContext)) as Record); - scope.setFingerprint(["rees-analyzer-degraded", analyzerTag]); - scope.setTag("event", "rees_analyzer_degraded"); - scope.setTag("analyzer", analyzerTag); - scope.setTag("repo", sentryTagValue(context.repoFullName) ?? "unknown"); - scope.setTag("pullNumber", sentryTagValue(context.prNumber) ?? "unknown"); - if (headShaTag) scope.setTag("headShaPrefix", headShaTag); - if (timeoutTag) scope.setTag("timeoutMs", timeoutTag); - if (releaseTag) scope.setTag("release", releaseTag); - const analyzerStatusTag = sentryTagValue(context.analyzerStatus); - const profileTag = sentryTagValue(context.profile); - const costClassTag = sentryTagValue(context.costClass); - const responseReserveTag = sentryTagValue(context.responseReserveMs); - const partialStatusTag = sentryTagValue(context.partialStatus); - const phaseTag = sentryTagValue(context.phase); - const endpointCategoryTag = sentryTagValue(context.endpointCategory); - const externalFailureReasonTag = sentryTagValue(context.externalFailureReason); - const endpointTag = sentryTagValue(context.githubEndpointCategory); - const requestIdTag = sentryTagValue(context.requestId); - const traceIdTag = sentryTagValue(context.traceId); - const cacheHitsTag = sentryTagValue(context.cacheHits); - const cacheMissesTag = sentryTagValue(context.cacheMisses); - if (analyzerStatusTag) scope.setTag("analyzerStatus", analyzerStatusTag); - if (profileTag) scope.setTag("profile", profileTag); - if (costClassTag) scope.setTag("costClass", costClassTag); - if (responseReserveTag) scope.setTag("responseReserveMs", responseReserveTag); - if (partialStatusTag) scope.setTag("partialStatus", partialStatusTag); - if (phaseTag) scope.setTag("phase", phaseTag); - if (endpointCategoryTag) scope.setTag("endpointCategory", endpointCategoryTag); - if (externalFailureReasonTag) scope.setTag("externalFailureReason", externalFailureReasonTag); - if (endpointTag) scope.setTag("githubEndpointCategory", endpointTag); - if (requestIdTag) scope.setTag("requestId", requestIdTag); - if (traceIdTag) scope.setTag("traceId", traceIdTag); - if (cacheHitsTag) scope.setTag("cacheHits", cacheHitsTag); - if (cacheMissesTag) scope.setTag("cacheMisses", cacheMissesTag); - scope.setTag("environment", sentryTagValue(activeEnvironment) ?? "production"); - Sentry!.captureException(error instanceof Error ? error : new Error(String(error))); + }, + fingerprint: ["rees-analyzer-degraded", context.analyzer], + tags: { + event: "rees_analyzer_degraded", + analyzer: context.analyzer, + repo: context.repoFullName, + pullNumber: context.prNumber, + }, }); } diff --git a/review-enrichment/src/server.ts b/review-enrichment/src/server.ts index c981b8018..4bc260a7e 100644 --- a/review-enrichment/src/server.ts +++ b/review-enrichment/src/server.ts @@ -16,7 +16,8 @@ import { readEnrichRequestText, } from "./request-guardrails.js"; import { - captureError, + captureRouteError, + captureUnhandledError, flushSentry, initSentry, resolveSentryEnvironment, @@ -46,7 +47,7 @@ app.get("/health", (c) => app.get("/ready", (c) => c.json({ ready: true })); app.onError((error, c) => { - captureError(error, { method: c.req.method, path: c.req.path }); + captureRouteError(error, { method: c.req.method, route: c.req.path }); return c.json({ error: "internal_error" }, 500); }); @@ -76,11 +77,11 @@ serve({ fetch: app.fetch, port }, (info) => { }); process.on("unhandledRejection", (reason) => { - captureError(reason, { event: "unhandled_rejection" }); + captureUnhandledError(reason, { event: "rees_unhandled_rejection" }); }); process.on("uncaughtException", (error) => { - captureError(error, { event: "uncaught_exception" }); + captureUnhandledError(error, { event: "rees_uncaught_exception" }); void flushSentry().finally(() => process.exit(1)); }); diff --git a/review-enrichment/src/upload-sourcemaps.ts b/review-enrichment/src/upload-sourcemaps.ts index c715652cb..1fd46a244 100644 --- a/review-enrichment/src/upload-sourcemaps.ts +++ b/review-enrichment/src/upload-sourcemaps.ts @@ -3,7 +3,13 @@ import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; import { dirname, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { resolveReesSentryRelease, resolveSentryEnvironment } from "./sentry.js"; +import { + captureSourcemapUploadFailure, + flushSentry, + initSentry, + resolveReesSentryRelease, + resolveSentryEnvironment, +} from "./sentry.js"; type RunOptions = { allowExistingRelease?: boolean; @@ -127,6 +133,7 @@ function runReleaseValidation(release: string, fields: { sha?: string; deployNam } async function main(): Promise { + await initSentry(process.env).catch(() => false); const release = resolveReesSentryRelease(process.env); const required = { SENTRY_AUTH_TOKEN: nonBlank(process.env.SENTRY_AUTH_TOKEN), @@ -192,6 +199,13 @@ async function main(): Promise { log("rees_sentry_sourcemap_upload_complete", { release }); return 0; } catch (error) { + captureSourcemapUploadFailure(error, { + release, + railwayDeploymentId: nonBlank(process.env.RAILWAY_DEPLOYMENT_ID), + strict, + sha: nonBlank(process.env.SENTRY_COMMIT_SHA) ?? nonBlank(process.env.RAILWAY_GIT_COMMIT_SHA), + }); + await flushSentry(); warn("rees_sentry_sourcemap_upload_failed", { release, message: error instanceof Error ? error.message : String(error), diff --git a/review-enrichment/test/sentry-degradation.test.ts b/review-enrichment/test/sentry-degradation.test.ts index 75195b4b5..642226003 100644 --- a/review-enrichment/test/sentry-degradation.test.ts +++ b/review-enrichment/test/sentry-degradation.test.ts @@ -4,6 +4,9 @@ import test, { afterEach } from "node:test"; import { buildBrief } from "../dist/brief.js"; import { captureAnalyzerDegradation, + captureRouteError, + captureSourcemapUploadFailure, + captureUnhandledError, resetSentryForTest, setSentryForTest, } from "../dist/sentry.js"; @@ -76,8 +79,6 @@ test("captureAnalyzerDegradation tags and fingerprints sanitized analyzer failur assert.equal(sentry.tags.analyzer, "dependency"); assert.equal(sentry.tags.repo, "JSONbored/gittensory"); assert.equal(sentry.tags.pullNumber, "7"); - assert.equal(sentry.tags.headShaPrefix, "abc123"); - assert.equal(sentry.tags.timeoutMs, "8000"); assert.equal(sentry.tags.release, "gittensory-rees@test"); assert.equal(sentry.tags.environment, "test"); assert.equal(sentry.captured[0].message, "registry timeout"); @@ -114,7 +115,6 @@ test("captureAnalyzerDegradation filters tag values before sending them", () => assert.deepEqual(sentry.fingerprints, [["rees-analyzer-degraded", "[Filtered]"]]); assert.equal(sentry.tags.analyzer, "[Filtered]"); assert.equal(sentry.tags.repo, "JSONbored/[Filtered]"); - assert.equal(sentry.tags.headShaPrefix, "[Filtered]"); }); test("captureAnalyzerDegradation attaches safe attribution context for history failures", () => { @@ -161,20 +161,6 @@ test("captureAnalyzerDegradation attaches safe attribution context for history f assert.equal(sentry.tags.analyzer, "history"); assert.equal(sentry.tags.repo, "JSONbored/metagraphed"); assert.equal(sentry.tags.pullNumber, "2359"); - assert.equal(sentry.tags.headShaPrefix, "abcdef123456"); - assert.equal(sentry.tags.timeoutMs, "7000"); - assert.equal(sentry.tags.analyzerStatus, "degraded"); - assert.equal(sentry.tags.profile, "balanced"); - assert.equal(sentry.tags.costClass, "github-heavy"); - assert.equal(sentry.tags.responseReserveMs, "750"); - assert.equal(sentry.tags.partialStatus, "partial"); - assert.equal(sentry.tags.phase, "similar_past_prs"); - assert.equal(sentry.tags.endpointCategory, "github-commit-pulls"); - assert.equal(sentry.tags.externalFailureReason, "timeout"); - assert.equal(sentry.tags.githubEndpointCategory, "commit_pulls"); - assert.equal(sentry.tags.cacheHits, "4"); - assert.equal(sentry.tags.cacheMisses, "9"); - assert.equal(sentry.tags.requestId, "req-123"); const analyzerContext = sentry.contexts.rees_analyzer as Record; assert.deepEqual(analyzerContext, { event: "rees_analyzer_degraded", @@ -252,12 +238,80 @@ test("buildBrief stays fail-open and captures a degraded analyzer", async () => assert.equal(sentry.tags.analyzer, "dependency"); assert.equal(sentry.tags.repo, "JSONbored/gittensory"); assert.equal(sentry.tags.pullNumber, "42"); - assert.equal(sentry.tags.headShaPrefix, "head-sha"); - const capturedTimeoutMs = Number(sentry.tags.timeoutMs); + assert.equal(sentry.tags.event, "rees_analyzer_degraded"); + const analyzerContext = sentry.contexts.rees_analyzer as Record; + const capturedTimeoutMs = Number(analyzerContext.timeoutMs); assert.ok(capturedTimeoutMs > 0); assert.ok(capturedTimeoutMs <= 200); }); +test("captureRouteError applies the route-level fingerprint and allowlisted tags", () => { + const sentry = sentryHarness(); + + captureRouteError(new Error("boom"), { + route: "/v1/enrich", + method: "POST", + }); + + assert.deepEqual(sentry.levels, ["error"]); + assert.deepEqual(sentry.fingerprints, [["rees-route-error", "/v1/enrich", "POST"]]); + assert.equal(sentry.tags.event, "rees_route_error"); + assert.equal(sentry.tags.route, "/v1/enrich"); + assert.equal(sentry.tags.method, "POST"); + assert.equal(sentry.tags.release, "gittensory-rees@test"); + assert.equal(sentry.tags.environment, "test"); + assert.deepEqual(sentry.contexts.rees_route, { + event: "rees_route_error", + route: "/v1/enrich", + method: "POST", + release: "gittensory-rees@test", + environment: "test", + }); +}); + +test("captureUnhandledError fingerprints process-level failures by event class", () => { + const sentry = sentryHarness(); + + captureUnhandledError(new Error("kaboom"), { event: "rees_uncaught_exception" }); + + assert.deepEqual(sentry.fingerprints, [["rees-process-error", "rees_uncaught_exception"]]); + assert.equal(sentry.tags.event, "rees_uncaught_exception"); + assert.equal(sentry.tags.release, "gittensory-rees@test"); + assert.equal(sentry.tags.environment, "test"); + assert.deepEqual(sentry.contexts.rees_process, { + event: "rees_uncaught_exception", + release: "gittensory-rees@test", + environment: "test", + }); +}); + +test("captureSourcemapUploadFailure applies stable upload grouping and safe tags", () => { + const sentry = sentryHarness(); + + captureSourcemapUploadFailure(new Error("upload failed"), { + release: "gittensory-rees@test", + railwayDeploymentId: "railway-deploy-123", + strict: true, + sha: "abcdef1234567890", + stage: "upload", + }); + + assert.deepEqual(sentry.fingerprints, [["rees-sourcemap-upload-failed"]]); + assert.equal(sentry.tags.event, "rees_sourcemap_upload_failed"); + assert.equal(sentry.tags.release, "gittensory-rees@test"); + assert.equal(sentry.tags.environment, "test"); + assert.equal(sentry.tags.railwayDeploymentId, "railway-deploy-123"); + assert.deepEqual(sentry.contexts.rees_sourcemap_upload, { + event: "rees_sourcemap_upload_failed", + release: "gittensory-rees@test", + railwayDeploymentId: "railway-deploy-123", + strict: true, + sha: "abcdef1234567890", + stage: "upload", + environment: "test", + }); +}); + test("buildBrief normalizes unsafe analyzer partial reasons before response telemetry", async () => { const fakeToken = ["ghp", "abcdefghijklmnopqrstuvwxyz1234567890"].join("_");