From 647a40e242690047217847da1cd5bdada47f4fea Mon Sep 17 00:00:00 2001 From: Aiden Bai Date: Mon, 10 Aug 2026 23:53:33 +0000 Subject: [PATCH 1/5] feat(cli): collect minimized rule evidence --- .changeset/bright-rules-report.md | 5 ++ packages/react-doctor/README.md | 9 ++- .../react-doctor/src/cli/commands/scan.ts | 12 +++ packages/react-doctor/src/cli/index.ts | 6 +- .../utils/anonymize-diagnostic-evidence.ts | 65 ++++++++++++++++ .../src/cli/utils/cli-state-store.ts | 2 + .../src/cli/utils/collect-rule-evidence.ts | 56 ++++++++++++++ .../react-doctor/src/cli/utils/constants.ts | 6 ++ .../src/cli/utils/record-rule-evidence.ts | 37 +++++++++ .../src/cli/utils/render-and-record-scan.ts | 7 ++ .../cli/utils/scan-result-cache-lifecycle.ts | 2 + .../src/cli/utils/telemetry-disclosure.ts | 36 +++++++++ .../anonymize-diagnostic-evidence.test.ts | 49 ++++++++++++ .../tests/collect-rule-evidence.test.ts | 76 +++++++++++++++++++ .../tests/run-scan-command.test.ts | 7 ++ .../tests/telemetry-disclosure.test.ts | 58 ++++++++++++++ 16 files changed, 426 insertions(+), 7 deletions(-) create mode 100644 .changeset/bright-rules-report.md create mode 100644 packages/react-doctor/src/cli/utils/anonymize-diagnostic-evidence.ts create mode 100644 packages/react-doctor/src/cli/utils/collect-rule-evidence.ts create mode 100644 packages/react-doctor/src/cli/utils/record-rule-evidence.ts create mode 100644 packages/react-doctor/src/cli/utils/telemetry-disclosure.ts create mode 100644 packages/react-doctor/tests/anonymize-diagnostic-evidence.test.ts create mode 100644 packages/react-doctor/tests/collect-rule-evidence.test.ts create mode 100644 packages/react-doctor/tests/telemetry-disclosure.test.ts diff --git a/.changeset/bright-rules-report.md b/.changeset/bright-rules-report.md new file mode 100644 index 0000000000..59b6684067 --- /dev/null +++ b/.changeset/bright-rules-report.md @@ -0,0 +1,5 @@ +--- +"react-doctor": patch +--- + +Collect minimized diagnostic source patterns in default-on telemetry to help fix false positives and false negatives, with a one-time interactive disclosure and `--no-telemetry` opt-out. diff --git a/packages/react-doctor/README.md b/packages/react-doctor/README.md index 78e1f545c8..ca73d3cc55 100644 --- a/packages/react-doctor/README.md +++ b/packages/react-doctor/README.md @@ -57,17 +57,18 @@ You can configure which rules to run and how to run them in `doctor.config.ts`. ## Telemetry -The CLI reports crashes, basic run traces, and anonymous usage counters to [Sentry](https://sentry.io/) to help us fix bugs and prioritize work. +Telemetry is on by default. The CLI sends crash reports to [Sentry](https://sentry.io/) and usage telemetry to [Axiom](https://axiom.co/). We use this data to operate React Doctor and improve its diagnostic rules. We collect: - Environment: CLI version, platform, Node version - Invocation: which command, package manager, and run context (whether it's local vs. CI vs. coding agent) -- Project shape: framework, React version, TypeScript, project size (NO file contents) -- Rules fired: rule names and counts only (e.g. `react-doctor/no-array-index-as-key`) (NO code or specific findings) +- Project shape: framework, React version, TypeScript, and project size +- Rules fired: rule names and counts (e.g. `react-doctor/no-array-index-as-key`) +- Rule evidence: minimized token patterns from source spans that trigger diagnostics. React Doctor replaces identifier names and literal contents, removes comments and file paths, and limits the number and size of patterns. It does not collect complete source files. We use these patterns to fix false positives, which are incorrect diagnostics, and false negatives, which are issues that rules miss - De-minified React Doctor CLI stack traces -To opt out, run: `npx react-doctor@latest --no-telemetry` +To disable telemetry for a run: `npx react-doctor@latest --no-telemetry` ## Contributing diff --git a/packages/react-doctor/src/cli/commands/scan.ts b/packages/react-doctor/src/cli/commands/scan.ts index 2089c3a911..10995a4f81 100644 --- a/packages/react-doctor/src/cli/commands/scan.ts +++ b/packages/react-doctor/src/cli/commands/scan.ts @@ -9,6 +9,7 @@ import { recordCount } from "../utils/record-metric.js"; import { resolveCliInspectOptions } from "../utils/resolve-cli-inspect-options.js"; import { warnDeprecatedDiff } from "../utils/resolve-scope.js"; import { shouldUseTui } from "../utils/should-use-tui.js"; +import { showTelemetryDisclosureIfNeeded } from "../utils/telemetry-disclosure.js"; import { validateModeFlags } from "../utils/validate-mode-flags.js"; import { warnDeprecatedFailOn } from "../utils/warn-deprecated-fail-on.js"; @@ -33,6 +34,16 @@ export const runScanCommand = async (input: RunScanCommandInput): Promise if (!shouldUseTui(tuiEnvironment)) { await inspectAction(input.directory, input.flags, input.invocationCommand); + showTelemetryDisclosureIfNeeded({ + isInteractive: + !tuiEnvironment.isNonInteractiveEnvironment && + tuiEnvironment.stdinIsTty && + tuiEnvironment.stdoutIsTty && + input.flags.json !== true && + input.flags.jsonCompact !== true && + input.flags.jsonOut === undefined && + input.flags.score !== true, + }); return; } @@ -53,4 +64,5 @@ export const runScanCommand = async (input: RunScanCommandInput): Promise flags: input.flags, }); if (shouldFail) process.exitCode = 1; + showTelemetryDisclosureIfNeeded({ isInteractive: true }); }; diff --git a/packages/react-doctor/src/cli/index.ts b/packages/react-doctor/src/cli/index.ts index b0414e9ec5..7efa78e5cf 100644 --- a/packages/react-doctor/src/cli/index.ts +++ b/packages/react-doctor/src/cli/index.ts @@ -203,7 +203,7 @@ const program = new Command() "scan source files listed in a newline-delimited changed-files file", ).hideHelp(), ) - .option("--no-score", "skip the score API, the share URL, and crash reporting") + .option("--no-score", "skip the score API, the share URL, and all telemetry") .addOption( new Option( "--category ", @@ -212,7 +212,7 @@ const program = new Command() ) .option( "--no-telemetry", - "alias for --no-score (skip the score API, share URL, and crash reporting)", + "disable all telemetry, including de-identified code patterns (also skips the score API and share URL)", ) .option( "--staged", @@ -487,7 +487,7 @@ program .option("--no-dead-code", "skip dead-code analysis") .option("--no-supply-chain", "skip the dependency supply-chain scan") .option("--score", "only print the numeric score (for scripts and CI)") - .option("--no-score", "skip the score API, the share URL, and crash reporting") + .option("--no-score", "skip the score API, the share URL, and all telemetry") .option("--no-cache", "disable all scan caches for this run") .option("--max-duration ", MAX_DURATION_OPTION_DESCRIPTION) .option("-p, --project ", "scan specific workspace projects (comma-separated, or *)") diff --git a/packages/react-doctor/src/cli/utils/anonymize-diagnostic-evidence.ts b/packages/react-doctor/src/cli/utils/anonymize-diagnostic-evidence.ts new file mode 100644 index 0000000000..2033082777 --- /dev/null +++ b/packages/react-doctor/src/cli/utils/anonymize-diagnostic-evidence.ts @@ -0,0 +1,65 @@ +import ts from "typescript"; +import { RULE_EVIDENCE_MAX_TOKEN_COUNT } from "./constants.js"; + +export interface AnonymizedDiagnosticEvidence { + readonly pattern: string; + readonly tokenCount: number; + readonly truncated: boolean; +} + +export const anonymizeDiagnosticEvidence = (evidence: string): AnonymizedDiagnosticEvidence => { + const scanner = ts.createScanner(ts.ScriptTarget.Latest, true, ts.LanguageVariant.JSX, evidence); + const identifierIndexes = new Map(); + const tokens: string[] = []; + let tokenCount = 0; + let tokenKind = scanner.scan(); + + while (tokenKind !== ts.SyntaxKind.EndOfFileToken) { + tokenCount += 1; + if (tokens.length < RULE_EVIDENCE_MAX_TOKEN_COUNT) { + let token: string; + if (tokenKind === ts.SyntaxKind.Identifier || tokenKind === ts.SyntaxKind.PrivateIdentifier) { + const identifier = scanner.getTokenText(); + const identifierIndex = identifierIndexes.get(identifier) ?? identifierIndexes.size + 1; + identifierIndexes.set(identifier, identifierIndex); + token = `identifier_${identifierIndex}`; + } else if ( + tokenKind === ts.SyntaxKind.StringLiteral || + tokenKind === ts.SyntaxKind.JsxText || + tokenKind === ts.SyntaxKind.JsxTextAllWhiteSpaces + ) { + token = "string_literal"; + } else if (tokenKind === ts.SyntaxKind.NumericLiteral) { + token = "number_literal"; + } else if ( + tokenKind === ts.SyntaxKind.TrueKeyword || + tokenKind === ts.SyntaxKind.FalseKeyword + ) { + token = "boolean_literal"; + } else if (tokenKind === ts.SyntaxKind.NullKeyword) { + token = "null_literal"; + } else if (tokenKind === ts.SyntaxKind.BigIntLiteral) { + token = "bigint_literal"; + } else if (tokenKind === ts.SyntaxKind.RegularExpressionLiteral) { + token = "regular_expression_literal"; + } else if ( + tokenKind === ts.SyntaxKind.NoSubstitutionTemplateLiteral || + tokenKind === ts.SyntaxKind.TemplateHead || + tokenKind === ts.SyntaxKind.TemplateMiddle || + tokenKind === ts.SyntaxKind.TemplateTail + ) { + token = "template_literal"; + } else { + token = ts.tokenToString(tokenKind) ?? `syntax_${tokenKind}`; + } + tokens.push(token); + } + tokenKind = scanner.scan(); + } + + return { + pattern: tokens.join(" "), + tokenCount, + truncated: tokenCount > RULE_EVIDENCE_MAX_TOKEN_COUNT, + }; +}; diff --git a/packages/react-doctor/src/cli/utils/cli-state-store.ts b/packages/react-doctor/src/cli/utils/cli-state-store.ts index d21500dd54..0c670b2355 100644 --- a/packages/react-doctor/src/cli/utils/cli-state-store.ts +++ b/packages/react-doctor/src/cli/utils/cli-state-store.ts @@ -37,11 +37,13 @@ export const INITIAL_LIFECYCLE_VERSION = 1; // surface kind scope id / migration id wired in // ──────────────────── ───────── ─────── ────────────────── ──────────────────────── // first-run onboarding gate global onboarding onboarding-state.ts +// telemetry disclosure gate global telemetry-disclosure telemetry-disclosure.ts // "add to CI?" pitch gate project ci-pitch ci-prompt-decision.ts // @v1 → @v2 offer gate project action-upgrade-v2 action-upgrade-prompt.ts // agent install hint gate project setup-hint prompt-install-setup.ts // config json → ts migration project config-json-to-ts cli-migrations.ts export const ONBOARDING_EVENT = "onboarding"; +export const TELEMETRY_DISCLOSURE_EVENT = "telemetry-disclosure"; export const CI_PITCH_EVENT = "ci-pitch"; export const ACTION_UPGRADE_EVENT = "action-upgrade-v2"; export const SETUP_HINT_EVENT = "setup-hint"; diff --git a/packages/react-doctor/src/cli/utils/collect-rule-evidence.ts b/packages/react-doctor/src/cli/utils/collect-rule-evidence.ts new file mode 100644 index 0000000000..cbf44f5a84 --- /dev/null +++ b/packages/react-doctor/src/cli/utils/collect-rule-evidence.ts @@ -0,0 +1,56 @@ +import { getDiagnosticRuleIdentity } from "@react-doctor/core"; +import type { Diagnostic } from "@react-doctor/core"; +import { anonymizeDiagnosticEvidence } from "./anonymize-diagnostic-evidence.js"; +import { + RULE_EVIDENCE_MAX_DIAGNOSTIC_COUNT, + RULE_EVIDENCE_MAX_PER_RULE_COUNT, +} from "./constants.js"; +import { createDiagnosticEvidenceReader } from "./read-diagnostic-evidence.js"; + +export interface RuleEvidenceRecord { + readonly category: string; + readonly fileContext: string; + readonly pattern: string; + readonly plugin: string; + readonly rule: string; + readonly severity: string; + readonly tokenCount: number; + readonly truncated: boolean; +} + +export const collectRuleEvidence = ( + directory: string, + diagnostics: ReadonlyArray, +): RuleEvidenceRecord[] => { + const readEvidence = createDiagnosticEvidenceReader(directory); + const evidenceRecords: RuleEvidenceRecord[] = []; + const patternKeys = new Set(); + const ruleCounts = new Map(); + + for (const diagnostic of diagnostics) { + if (evidenceRecords.length >= RULE_EVIDENCE_MAX_DIAGNOSTIC_COUNT) break; + if (diagnostic.plugin !== "react-doctor") continue; + const { ruleKey, category } = getDiagnosticRuleIdentity(diagnostic); + if ((ruleCounts.get(ruleKey) ?? 0) >= RULE_EVIDENCE_MAX_PER_RULE_COUNT) continue; + const evidence = readEvidence(diagnostic); + if (evidence === null) continue; + const anonymizedEvidence = anonymizeDiagnosticEvidence(evidence); + if (anonymizedEvidence.pattern === "") continue; + const patternKey = `${ruleKey}\0${anonymizedEvidence.pattern}`; + if (patternKeys.has(patternKey)) continue; + patternKeys.add(patternKey); + ruleCounts.set(ruleKey, (ruleCounts.get(ruleKey) ?? 0) + 1); + evidenceRecords.push({ + category, + fileContext: diagnostic.fileContext ?? "production", + pattern: anonymizedEvidence.pattern, + plugin: diagnostic.plugin, + rule: ruleKey, + severity: diagnostic.severity, + tokenCount: anonymizedEvidence.tokenCount, + truncated: anonymizedEvidence.truncated, + }); + } + + return evidenceRecords; +}; diff --git a/packages/react-doctor/src/cli/utils/constants.ts b/packages/react-doctor/src/cli/utils/constants.ts index f2149e5e00..b94c38e466 100644 --- a/packages/react-doctor/src/cli/utils/constants.ts +++ b/packages/react-doctor/src/cli/utils/constants.ts @@ -220,6 +220,11 @@ export const NANOSECONDS_PER_MILLISECOND = 1_000_000n; // lose everything recorded since startup. export const LSP_TELEMETRY_EXPORT_INTERVAL_MS = 60_000; +export const RULE_EVIDENCE_SCHEMA_VERSION = 1; +export const RULE_EVIDENCE_MAX_DIAGNOSTIC_COUNT = 24; +export const RULE_EVIDENCE_MAX_PER_RULE_COUNT = 3; +export const RULE_EVIDENCE_MAX_TOKEN_COUNT = 160; + export const AXIOM_TRACES_DATASET = "react-doctor"; export const AXIOM_METRICS_DATASET = "react-doctor-metrics"; @@ -290,6 +295,7 @@ export const METRIC = { scanNoReactDetected: "scan.no_react_detected", baselineDegraded: "baseline.degraded", ruleFired: "rule.fired", + ruleEvidenceCollected: "rule.evidence_collected", // Rule-rejection telemetry, both keyed by `rule` + `source` attributes: // `rule.disabled` counts one per scan per config-off rule (`rules: "off"` / // `ignore.rules` — the former never fires, so this is its only signal); diff --git a/packages/react-doctor/src/cli/utils/record-rule-evidence.ts b/packages/react-doctor/src/cli/utils/record-rule-evidence.ts new file mode 100644 index 0000000000..e663bfeb5f --- /dev/null +++ b/packages/react-doctor/src/cli/utils/record-rule-evidence.ts @@ -0,0 +1,37 @@ +import type { Diagnostic } from "@react-doctor/core"; +import { collectRuleEvidence } from "./collect-rule-evidence.js"; +import { METRIC, NANOSECONDS_PER_MILLISECOND, RULE_EVIDENCE_SCHEMA_VERSION } from "./constants.js"; +import { recordCount } from "./record-metric.js"; +import type { RunRootSpan } from "./with-run-span.js"; + +export interface RecordRuleEvidenceInput { + readonly diagnostics: ReadonlyArray; + readonly directory: string; + readonly rootSpan: RunRootSpan; +} + +export const recordRuleEvidence = (input: RecordRuleEvidenceInput): void => { + if (input.rootSpan === undefined) return; + try { + for (const evidence of collectRuleEvidence(input.directory, input.diagnostics)) { + input.rootSpan.event("rule.evidence", BigInt(Date.now()) * NANOSECONDS_PER_MILLISECOND, { + "evidence.schemaVersion": RULE_EVIDENCE_SCHEMA_VERSION, + "evidence.outcome": "diagnostic", + "evidence.pattern": evidence.pattern, + "evidence.tokenCount": evidence.tokenCount, + "evidence.truncated": evidence.truncated, + "evidence.fileContext": evidence.fileContext, + rule: evidence.rule, + plugin: evidence.plugin, + category: evidence.category, + severity: evidence.severity, + }); + recordCount(METRIC.ruleEvidenceCollected, 1, { + rule: evidence.rule, + plugin: evidence.plugin, + category: evidence.category, + severity: evidence.severity, + }); + } + } catch {} +}; diff --git a/packages/react-doctor/src/cli/utils/render-and-record-scan.ts b/packages/react-doctor/src/cli/utils/render-and-record-scan.ts index f5f8003c50..09f1f1e612 100644 --- a/packages/react-doctor/src/cli/utils/render-and-record-scan.ts +++ b/packages/react-doctor/src/cli/utils/render-and-record-scan.ts @@ -11,12 +11,14 @@ import { type InspectExecutionCacheStats, } from "./finalize-inspect-result.js"; import { makeNoopConsole } from "./noop-console.js"; +import { recordRuleEvidence } from "./record-rule-evidence.js"; import { recordScanMetrics } from "./record-scan-metrics.js"; import { resolveWorkerTelemetry } from "./resolve-worker-telemetry.js"; import type { CachedScanPayload } from "./scan-result-cache-payload.js"; import type { RunRootSpan } from "./with-run-span.js"; export interface RenderAndRecordScanInput { + readonly directory: string; readonly payload: CachedScanPayload; readonly options: ResolvedInspectOptions; readonly startTime: number; @@ -106,6 +108,11 @@ export const renderAndRecordScan = async ( userConfig: input.payload.userConfig, suppressedRuleCounts: input.payload.suppressedRuleCounts ?? [], }); + recordRuleEvidence({ + diagnostics: result.diagnostics, + directory: input.directory, + rootSpan: input.rootSpan, + }); recordRunEvent(input.rootSpan, { ...buildRunEventConfig(input.options, input.payload.userConfig, resolvedWorkerCount), result, diff --git a/packages/react-doctor/src/cli/utils/scan-result-cache-lifecycle.ts b/packages/react-doctor/src/cli/utils/scan-result-cache-lifecycle.ts index 868ab303d2..d706ed1bf6 100644 --- a/packages/react-doctor/src/cli/utils/scan-result-cache-lifecycle.ts +++ b/packages/react-doctor/src/cli/utils/scan-result-cache-lifecycle.ts @@ -67,6 +67,7 @@ export const createScanResultCacheLifecycle = ( const baselineDegraded = Boolean(input.options.baseline) && isDiffMode && cachedPayload.baselineDelta === undefined; return renderAndRecordScan({ + directory: input.directory, payload: cachedPayload, options: input.options, startTime: input.startTime, @@ -86,6 +87,7 @@ export const createScanResultCacheLifecycle = ( scanResultCache.store(cacheKey, completion.payload); } return renderAndRecordScan({ + directory: input.directory, payload: completion.payload, options: input.options, startTime: input.startTime, diff --git a/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts b/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts new file mode 100644 index 0000000000..eca4f2e6b9 --- /dev/null +++ b/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts @@ -0,0 +1,36 @@ +import { type CliStateOptions, TELEMETRY_DISCLOSURE_EVENT } from "./cli-state-store.js"; +import { type Gate, isGatePending, recordGate } from "./cli-lifecycle.js"; +import { cliLogger } from "./cli-logger.js"; +import { isTelemetryEnabled } from "./is-telemetry-enabled.js"; + +const TELEMETRY_DISCLOSURE_GATE: Gate = { + id: TELEMETRY_DISCLOSURE_EVENT, + scope: "global", +}; + +export const TELEMETRY_DISCLOSURE_LINES = [ + "React Doctor telemetry is on by default.", + "Telemetry includes usage data and minimized, de-identified code patterns.", + "We use these patterns to fix false positives, which are incorrect diagnostics.", + "We also use them to fix false negatives, which are issues that rules miss.", + "React Doctor does not collect complete source files.", + "Run with --no-telemetry to disable telemetry.", +]; + +export interface ShowTelemetryDisclosureInput { + readonly isInteractive: boolean; + readonly store?: CliStateOptions; + readonly telemetryEnabled?: boolean; + readonly writeLine?: (line: string) => void; +} + +export const showTelemetryDisclosureIfNeeded = (input: ShowTelemetryDisclosureInput): boolean => { + if (!input.isInteractive || !(input.telemetryEnabled ?? isTelemetryEnabled())) return false; + if (!isGatePending(TELEMETRY_DISCLOSURE_GATE, {}, input.store)) return false; + const writeLine = input.writeLine ?? cliLogger.log; + writeLine(""); + for (const line of TELEMETRY_DISCLOSURE_LINES) writeLine(line); + writeLine(""); + recordGate(TELEMETRY_DISCLOSURE_GATE, { outcome: "seen" }, input.store); + return true; +}; diff --git a/packages/react-doctor/tests/anonymize-diagnostic-evidence.test.ts b/packages/react-doctor/tests/anonymize-diagnostic-evidence.test.ts new file mode 100644 index 0000000000..1c5405be96 --- /dev/null +++ b/packages/react-doctor/tests/anonymize-diagnostic-evidence.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vite-plus/test"; +import { anonymizeDiagnosticEvidence } from "../src/cli/utils/anonymize-diagnostic-evidence.js"; +import { RULE_EVIDENCE_MAX_TOKEN_COUNT } from "../src/cli/utils/constants.js"; + +describe("anonymizeDiagnosticEvidence", () => { + it("preserves syntax and binding relationships without source names or literal contents", () => { + const evidence = ` + // Customer-specific behavior + const customerEmail = "person@example.com"; + useEffect(() => setCustomer(customerEmail), [customerEmail]); + `; + + const result = anonymizeDiagnosticEvidence(evidence); + + expect(result.pattern).toContain("const identifier_1 = string_literal ;"); + expect(result.pattern).toContain("identifier_2 ( ( ) => identifier_3 ( identifier_1 )"); + expect(result.pattern).not.toContain("Customer-specific"); + expect(result.pattern).not.toContain("customerEmail"); + expect(result.pattern).not.toContain("person@example.com"); + expect(result.truncated).toBe(false); + }); + + it("uses stable placeholders for repeated identifiers", () => { + const result = anonymizeDiagnosticEvidence("value + value + other"); + + expect(result.pattern).toBe("identifier_1 + identifier_1 + identifier_2"); + }); + + it("removes primitive literal values", () => { + const result = anonymizeDiagnosticEvidence("[0, 1, 42, true, false, null]"); + + expect(result.pattern).toBe( + "[ number_literal , number_literal , number_literal , boolean_literal , boolean_literal , null_literal ]", + ); + }); + + it("bounds the exported pattern while retaining the original token count", () => { + const evidence = Array.from( + { length: RULE_EVIDENCE_MAX_TOKEN_COUNT + 10 }, + (_unused, index) => `identifier${index};`, + ).join(" "); + + const result = anonymizeDiagnosticEvidence(evidence); + + expect(result.tokenCount).toBeGreaterThan(RULE_EVIDENCE_MAX_TOKEN_COUNT); + expect(result.pattern.split(" ")).toHaveLength(RULE_EVIDENCE_MAX_TOKEN_COUNT); + expect(result.truncated).toBe(true); + }); +}); diff --git a/packages/react-doctor/tests/collect-rule-evidence.test.ts b/packages/react-doctor/tests/collect-rule-evidence.test.ts new file mode 100644 index 0000000000..91e4bc1285 --- /dev/null +++ b/packages/react-doctor/tests/collect-rule-evidence.test.ts @@ -0,0 +1,76 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { Diagnostic } from "@react-doctor/core"; +import { afterEach, beforeEach, describe, expect, it } from "vite-plus/test"; +import { collectRuleEvidence } from "../src/cli/utils/collect-rule-evidence.js"; + +const buildDiagnostic = (overrides: Partial = {}): Diagnostic => ({ + filePath: "src/example.tsx", + plugin: "react-doctor", + rule: "no-direct-set-state-in-use-effect", + severity: "warning", + message: "State update in an effect", + help: "Derive the value while rendering", + line: 2, + column: 1, + category: "State & Effects", + ...overrides, +}); + +describe("collectRuleEvidence", () => { + let directory: string; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), "react-doctor-rule-evidence-")); + fs.mkdirSync(path.join(directory, "src")); + fs.writeFileSync( + path.join(directory, "src/example.tsx"), + [ + "const customerName = 'private customer';", + "useEffect(() => setName(customerName), [customerName]);", + "const accountSecret = 'low-entropy-secret';", + "useEffect(() => setAccount({ value: accountSecret }), [accountSecret]);", + ].join("\n"), + ); + }); + + afterEach(() => { + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("collects bounded source patterns without file paths, identifiers, or literals", () => { + const [record] = collectRuleEvidence(directory, [buildDiagnostic()]); + + expect(record?.rule).toBe("react-doctor/no-direct-set-state-in-use-effect"); + expect(record?.pattern).toContain("identifier_1"); + expect(record?.pattern).not.toContain("example.tsx"); + expect(record?.pattern).not.toContain("customerName"); + expect(record?.pattern).not.toContain("private customer"); + }); + + it("deduplicates identical evidence for the same rule", () => { + const records = collectRuleEvidence(directory, [buildDiagnostic(), buildDiagnostic()]); + + expect(records).toHaveLength(1); + }); + + it("keeps distinct evidence without exposing its source text", () => { + const records = collectRuleEvidence(directory, [ + buildDiagnostic(), + buildDiagnostic({ line: 4 }), + ]); + + expect(records).toHaveLength(2); + expect(records[1]?.pattern).not.toContain("accountSecret"); + expect(records[1]?.pattern).not.toContain("low-entropy-secret"); + }); + + it("ignores diagnostics from third-party plugins", () => { + const records = collectRuleEvidence(directory, [ + buildDiagnostic({ plugin: "private-company-plugin" }), + ]); + + expect(records).toEqual([]); + }); +}); diff --git a/packages/react-doctor/tests/run-scan-command.test.ts b/packages/react-doctor/tests/run-scan-command.test.ts index 21a7bbddd1..91dc29b308 100644 --- a/packages/react-doctor/tests/run-scan-command.test.ts +++ b/packages/react-doctor/tests/run-scan-command.test.ts @@ -29,6 +29,10 @@ vi.mock("../src/cli/utils/should-use-tui.js", () => ({ shouldUseTui: vi.fn(() => true), })); +vi.mock("../src/cli/utils/telemetry-disclosure.js", () => ({ + showTelemetryDisclosureIfNeeded: vi.fn(), +})); + vi.mock("../src/cli/utils/resolve-scope.js", () => ({ warnDeprecatedDiff: vi.fn(), })); @@ -61,6 +65,7 @@ import { recordCount } from "../src/cli/utils/record-metric.js"; import { resolveCliInspectOptions } from "../src/cli/utils/resolve-cli-inspect-options.js"; import { warnDeprecatedDiff } from "../src/cli/utils/resolve-scope.js"; import { shouldUseTui } from "../src/cli/utils/should-use-tui.js"; +import { showTelemetryDisclosureIfNeeded } from "../src/cli/utils/telemetry-disclosure.js"; import { validateModeFlags } from "../src/cli/utils/validate-mode-flags.js"; import { warnDeprecatedFailOn } from "../src/cli/utils/warn-deprecated-fail-on.js"; @@ -117,6 +122,7 @@ describe("runScanCommand", () => { expect(warnDeprecatedFailOn).toHaveBeenCalledWith(flags, null); expect(warnDeprecatedDiff).toHaveBeenCalledWith(flags, null); expect(inspectAction).not.toHaveBeenCalled(); + expect(showTelemetryDisclosureIfNeeded).toHaveBeenCalledWith({ isInteractive: true }); }); it("uses headless output when the TUI gate rejects the environment or flags", async () => { @@ -129,6 +135,7 @@ describe("runScanCommand", () => { expect(runScanApp).not.toHaveBeenCalled(); expect(runProjectMigrations).not.toHaveBeenCalled(); expect(recordCount).not.toHaveBeenCalled(); + expect(showTelemetryDisclosureIfNeeded).toHaveBeenCalledWith({ isInteractive: false }); }); it("preserves the TUI scan exit code", async () => { diff --git a/packages/react-doctor/tests/telemetry-disclosure.test.ts b/packages/react-doctor/tests/telemetry-disclosure.test.ts new file mode 100644 index 0000000000..ea24826e88 --- /dev/null +++ b/packages/react-doctor/tests/telemetry-disclosure.test.ts @@ -0,0 +1,58 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vite-plus/test"; +import { + TELEMETRY_DISCLOSURE_LINES, + showTelemetryDisclosureIfNeeded, +} from "../src/cli/utils/telemetry-disclosure.js"; + +describe("showTelemetryDisclosureIfNeeded", () => { + let configDirectory: string; + + beforeEach(() => { + configDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "react-doctor-telemetry-notice-")); + }); + + afterEach(() => { + fs.rmSync(configDirectory, { recursive: true, force: true }); + }); + + it("shows the disclosure once for an interactive telemetry-enabled run", () => { + const lines: string[] = []; + const input = { + isInteractive: true, + store: { cwd: configDirectory }, + telemetryEnabled: true, + writeLine: (line: string) => lines.push(line), + }; + + expect(showTelemetryDisclosureIfNeeded(input)).toBe(true); + expect(lines).toEqual(["", ...TELEMETRY_DISCLOSURE_LINES, ""]); + expect(showTelemetryDisclosureIfNeeded(input)).toBe(false); + }); + + it("does not consume the disclosure during a headless run", () => { + const input = { + isInteractive: false, + store: { cwd: configDirectory }, + telemetryEnabled: true, + writeLine: (): void => {}, + }; + + expect(showTelemetryDisclosureIfNeeded(input)).toBe(false); + expect(showTelemetryDisclosureIfNeeded({ ...input, isInteractive: true })).toBe(true); + }); + + it("does not show or consume the disclosure when telemetry is disabled", () => { + const input = { + isInteractive: true, + store: { cwd: configDirectory }, + telemetryEnabled: false, + writeLine: (): void => {}, + }; + + expect(showTelemetryDisclosureIfNeeded(input)).toBe(false); + expect(showTelemetryDisclosureIfNeeded({ ...input, telemetryEnabled: true })).toBe(true); + }); +}); From 455590ab88308b42fd7efd7bf17253a76588d7f0 Mon Sep 17 00:00:00 2001 From: Aiden Bai Date: Tue, 11 Aug 2026 00:33:24 +0000 Subject: [PATCH 2/5] chore(skills): add telemetry reproer --- .agents/skills/reproer/SKILL.md | 93 +++++++++++++++++++++++ .agents/skills/reproer/agents/openai.yaml | 4 + 2 files changed, 97 insertions(+) create mode 100644 .agents/skills/reproer/SKILL.md create mode 100644 .agents/skills/reproer/agents/openai.yaml diff --git a/.agents/skills/reproer/SKILL.md b/.agents/skills/reproer/SKILL.md new file mode 100644 index 0000000000..160bfdfbe2 --- /dev/null +++ b/.agents/skills/reproer/SKILL.md @@ -0,0 +1,93 @@ +--- +name: reproer +description: Turn React Doctor `rule.evidence` telemetry events or de-identified token patterns into synthetic, parseable React/TypeScript repro hypotheses and fuzz cases. Use when investigating possible false positives or false negatives from an `evidence.pattern`, reconstructing a minimal rule-triggering shape, producing adversarial variants, or promoting a verified hypothesis into the React Doctor fuzz corpus. +--- + +# Reproer + +Generate synthetic programs that could produce the supplied token pattern. Treat every result as a hypothesis, never as recovered source. + +## Protect the privacy boundary + +- Never claim to reconstruct the original code. The mapping from source to evidence is many-to-one. +- Never search GitHub, the internet, or private repositories to identify a matching source unless the user explicitly requests that separate investigation. +- Invent neutral identifiers and literal values. Do not guess names, domains, secrets, UI copy, package names, or repository identity. +- Preserve an `identifier_N` equality relationship only within one candidate. Do not correlate placeholders across events. +- Keep the telemetry event out of committed fixtures. Commit only synthetic code written from the structural hypothesis. +- Describe reconstruction confidence as structural fidelity, not source fidelity. + +## Require the useful inputs + +Collect these fields when available: + +```text +rule +evidence.pattern +evidence.fileContext +evidence.tokenCount +evidence.truncated +category +severity +``` + +Require `rule` and `evidence.pattern`. Ask for either field if missing because the rule supplies semantics that the anonymized identifiers no longer carry. + +## Build repro hypotheses + +1. Locate and read the target rule, its metadata, focused tests, and any existing fuzz fixtures. Use `rg` with the short rule ID. +2. Read `.agents/skills/fuzz/SKILL.md` and `packages/fuzz/README.md` before running the harness or modifying its corpus. +3. Decode the pattern conservatively: + - Map each `identifier_N` to one neutral role name and reuse it within the candidate. + - Replace `string_literal`, `number_literal`, `boolean_literal`, `null_literal`, `bigint_literal`, `regular_expression_literal`, and `template_literal` with harmless canonical values. + - Preserve keywords, operators, delimiters, member access, calls, and identifier-equality relationships. + - Treat `syntax_N` as unknown until checking the matching TypeScript `SyntaxKind`. + - Add the smallest component, hook, import, or function wrapper needed to parse and exercise the rule. + - When `evidence.truncated` is true, close the program into a valid minimal shape without pretending the invented suffix matches the source. +4. Produce a small hypothesis matrix: + - **Nearest shape:** retain the observed token order with only parseability scaffolding. + - **Valid-context shape:** place the syntax in the most ordinary legitimate React or TypeScript context. + - **False-positive probe:** create a valid program where the suspicious shape is intentional or safe. + - **False-negative probes:** change one dimension at a time, such as aliasing, wrappers, optional chaining, parentheses, control flow, TypeScript syntax, JSX placement, or cross-statement data flow. +5. Generate three to eight candidates. Avoid a combinatorial matrix until one candidate reaches the rule's reporting path. + +## Verify instead of guessing + +Place initial candidates in a temporary directory or an existing focused test harness. Do not add corpus files before checking the verdict. + +For every candidate, record: + +- whether it parses; +- whether the target rule fires; +- the diagnostic span and message; +- the intended semantic verdict: valid, invalid, or uncertain; +- the mutation dimension that distinguishes it from the nearest shape. + +Do not call a case a false positive or false negative from telemetry alone. Confirm the rule contract and the program's semantics first. + +Run the narrowest focused rule test. After it passes, run the targeted fuzzer: + +```sh +FUZZ_RULE= FUZZ_STRICT=1 FUZZ_ITERATIONS=500 nr fuzz +``` + +Confirm that the target rule fires at least once. A silent run validates only early exits. + +## Promote verified cases + +- For a confirmed false positive, add a minimal focused valid case and `packages/fuzz/corpus/regressions/--.tsx` with `// verdict: pass`. +- For a confirmed true positive or liveness seed, add a minimal focused invalid case and use `packages/fuzz/corpus/targets/` when the harness needs a reporting-path seed. +- For a confirmed false negative, add the missing invalid case to the focused rule tests, fix the detector when requested, and add a generator snippet only when existing pools cannot produce the weakness. +- Use stable weakness names from the fuzz skill. Never label an unverified hypothesis as a regression. +- Run the focused test, `nr -C packages/fuzz test`, and replay the target rule after changing the corpus. + +## Report the result + +Return: + +1. the normalized evidence skeleton; +2. the synthetic candidates and their structural-fidelity notes; +3. the observed rule verdict for each candidate; +4. confirmed FP, FN, or liveness findings, clearly separated from hypotheses; +5. tests, fuzz commands, seeds, and files changed. + +State explicitly that the candidates do not recover identifiers, literals, comments, paths, or repository identity. diff --git a/.agents/skills/reproer/agents/openai.yaml b/.agents/skills/reproer/agents/openai.yaml new file mode 100644 index 0000000000..0790dad8dc --- /dev/null +++ b/.agents/skills/reproer/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Reproer" + short_description: "Create synthetic repros from token evidence" + default_prompt: "Use $reproer to turn this de-identified rule-evidence pattern into synthetic React Doctor fuzz cases." From 502d273e22e5b1f1fba8470e657862889f408c54 Mon Sep 17 00:00:00 2001 From: Aiden Bai Date: Tue, 11 Aug 2026 00:49:45 +0000 Subject: [PATCH 3/5] feat(cli): index rule evidence with scores --- .changeset/bright-rules-report.md | 2 +- packages/core/src/request-score.ts | 1 + packages/core/src/run-inspect.ts | 1 + packages/core/src/services/score.ts | 9 ++++- packages/core/src/types/index.ts | 1 + packages/core/src/types/run-inspect.ts | 5 ++- packages/core/src/types/score.ts | 14 +++++++ packages/core/tests/calculate-score.test.ts | 20 ++++++++++ packages/core/tests/run-inspect.test.ts | 38 +++++++++++++++++++ packages/react-doctor/README.md | 2 +- .../src/cli/utils/collect-rule-evidence.ts | 19 +++------- .../src/cli/utils/collect-score-evidence.ts | 28 ++++++++++++++ .../src/cli/utils/record-rule-evidence.ts | 37 ------------------ .../src/cli/utils/render-and-record-scan.ts | 6 --- .../src/cli/utils/telemetry-disclosure.ts | 1 + packages/react-doctor/src/inspect.ts | 5 +++ .../tests/collect-rule-evidence.test.ts | 1 + 17 files changed, 129 insertions(+), 61 deletions(-) create mode 100644 packages/react-doctor/src/cli/utils/collect-score-evidence.ts delete mode 100644 packages/react-doctor/src/cli/utils/record-rule-evidence.ts diff --git a/.changeset/bright-rules-report.md b/.changeset/bright-rules-report.md index 59b6684067..ef69b5f4ff 100644 --- a/.changeset/bright-rules-report.md +++ b/.changeset/bright-rules-report.md @@ -2,4 +2,4 @@ "react-doctor": patch --- -Collect minimized diagnostic source patterns in default-on telemetry to help fix false positives and false negatives, with a one-time interactive disclosure and `--no-telemetry` opt-out. +Collect minimized diagnostic source patterns with score telemetry to help fix false positives and false negatives, with score-level indexing, a one-time interactive disclosure, and a `--no-telemetry` opt-out. diff --git a/packages/core/src/request-score.ts b/packages/core/src/request-score.ts index 6051f40251..1a73caa232 100644 --- a/packages/core/src/request-score.ts +++ b/packages/core/src/request-score.ts @@ -84,6 +84,7 @@ const buildScoreRequestBody = ( return gzipSync( JSON.stringify({ diagnostics: sanitizeScoreDiagnostics(diagnostics), + ...(options.ruleEvidence !== undefined ? { ruleEvidence: options.ruleEvidence } : {}), ...buildScoreRequestMetadata(options.metadata), }), ); diff --git a/packages/core/src/run-inspect.ts b/packages/core/src/run-inspect.ts index 7ec1586531..b5f9545a11 100644 --- a/packages/core/src/run-inspect.ts +++ b/packages/core/src/run-inspect.ts @@ -890,6 +890,7 @@ export const runInspect = ( diagnostics: scoreDiagnostics, isCi: input.isCi, metadata: scoreMetadata, + ruleEvidence: input.collectScoreEvidence?.(scoreDiagnostics), }); const lintPartialFailures = yield* Ref.get(partialFailuresRef); const didSecurityScanFail = yield* Ref.get(securityScanFailedRef); diff --git a/packages/core/src/services/score.ts b/packages/core/src/services/score.ts index 85f300ebd9..67e7fef82f 100644 --- a/packages/core/src/services/score.ts +++ b/packages/core/src/services/score.ts @@ -3,13 +3,19 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient"; import * as HttpClient from "effect/unstable/http/HttpClient"; -import type { Diagnostic, ScoreRequestMetadata, ScoreResult } from "../types/index.js"; +import type { + Diagnostic, + ScoreRequestMetadata, + ScoreResult, + ScoreRuleEvidence, +} from "../types/index.js"; import { requestScore } from "../request-score.js"; interface ComputeInput { readonly diagnostics: ReadonlyArray; readonly isCi?: boolean; readonly metadata?: ScoreRequestMetadata; + readonly ruleEvidence?: ReadonlyArray; } export class Score extends Context.Service< @@ -27,6 +33,7 @@ export class Score extends Context.Service< requestScore(httpClient, input.diagnostics, { isCi: input.isCi, metadata: input.metadata, + ruleEvidence: input.ruleEvidence, }), ), }); diff --git a/packages/core/src/types/index.ts b/packages/core/src/types/index.ts index 0c6c70a2f8..c7f72c6948 100644 --- a/packages/core/src/types/index.ts +++ b/packages/core/src/types/index.ts @@ -66,6 +66,7 @@ export type { PromptMultiselectChoiceState, PromptMultiselectContext } from "./p // See that file for the duplication rationale. export type { CalculateScoreOptions, + ScoreRuleEvidence, ScoreRequestMetadata, ScoreResult, RulePriority, diff --git a/packages/core/src/types/run-inspect.ts b/packages/core/src/types/run-inspect.ts index 582063a715..fcc54aa849 100644 --- a/packages/core/src/types/run-inspect.ts +++ b/packages/core/src/types/run-inspect.ts @@ -3,7 +3,7 @@ import type { OxlintUnavailable, ReactDoctorErrorReason } from "../errors.js"; import type { DiagnosticSurface, ReactDoctorConfig } from "./config.js"; import type { Diagnostic, SourceFileEntry, SuppressedRuleCount } from "./diagnostic.js"; import type { ProjectInfo } from "./project-info.js"; -import type { ScoreRequestMetadata, ScoreResult } from "./score.js"; +import type { ScoreRequestMetadata, ScoreResult, ScoreRuleEvidence } from "./score.js"; export interface InspectInput { readonly directory: string; @@ -32,6 +32,9 @@ export interface InspectInput { readonly doctorVersion?: string; /** Random per-run id. */ readonly runId?: string; + readonly collectScoreEvidence?: ( + diagnostics: ReadonlyArray, + ) => ReadonlyArray; /** Enables best-effort authenticated local GitHub permission lookup for score metadata. */ readonly resolveLocalGithubViewerPermission?: boolean; /** diff --git a/packages/core/src/types/score.ts b/packages/core/src/types/score.ts index 09d806d2a4..e216930f1b 100644 --- a/packages/core/src/types/score.ts +++ b/packages/core/src/types/score.ts @@ -1,10 +1,24 @@ import type { ProjectInfo } from "./project-info.js"; +import type { Diagnostic, DiagnosticFileContext } from "./diagnostic.js"; export type RuleTier = "P0" | "P1" | "P2" | "P3"; export interface CalculateScoreOptions { isCi?: boolean; metadata?: ScoreRequestMetadata; + ruleEvidence?: ReadonlyArray; +} + +export interface ScoreRuleEvidence { + readonly schemaVersion: 1; + readonly category: string; + readonly fileContext: DiagnosticFileContext; + readonly pattern: string; + readonly plugin: string; + readonly rule: string; + readonly severity: Diagnostic["severity"]; + readonly tokenCount: number; + readonly truncated: boolean; } export interface ScoreRequestMetadata { diff --git a/packages/core/tests/calculate-score.test.ts b/packages/core/tests/calculate-score.test.ts index d4fd9c1dc3..8c92d512f1 100644 --- a/packages/core/tests/calculate-score.test.ts +++ b/packages/core/tests/calculate-score.test.ts @@ -82,6 +82,19 @@ describe("calculateScore", () => { githubActorAssociation: "CONTRIBUTOR", githubViewerPermission: "write", }, + ruleEvidence: [ + { + schemaVersion: 1, + category: "State & Effects", + fileContext: "production", + pattern: "identifier_1 ( )", + plugin: "react-doctor", + rule: "react-doctor/example-rule", + severity: "error", + tokenCount: 4, + truncated: false, + }, + ], }); expect(result).toEqual(apiScoreResponse); @@ -109,6 +122,13 @@ describe("calculateScore", () => { githubEventName: "pull_request", githubActorAssociation: "CONTRIBUTOR", githubViewerPermission: "write", + ruleEvidence: [ + { + schemaVersion: 1, + pattern: "identifier_1 ( )", + rule: "react-doctor/example-rule", + }, + ], }); }); diff --git a/packages/core/tests/run-inspect.test.ts b/packages/core/tests/run-inspect.test.ts index 76a4ac4da3..2429783f46 100644 --- a/packages/core/tests/run-inspect.test.ts +++ b/packages/core/tests/run-inspect.test.ts @@ -11,6 +11,7 @@ import type { Diagnostic, ProjectInfo, ReactDoctorConfig, + ScoreRuleEvidence, SourceFileEntry, } from "@react-doctor/core"; import { @@ -501,6 +502,43 @@ describe("runInspect — happy path", () => { expect(output.didDeadCodeFail).toBe(false); }); + it("attaches collected rule evidence to the score request", async () => { + const ruleEvidence: ScoreRuleEvidence = { + schemaVersion: 1, + category: "Correctness", + fileContext: "production", + pattern: "identifier_1 ( )", + plugin: "react-doctor", + rule: "react-doctor/no-derived-state", + severity: "error", + tokenCount: 4, + truncated: false, + }; + let receivedRuleEvidence: ReadonlyArray | undefined; + const scoreLayer = Layer.succeed( + Score, + Score.of({ + compute: (input) => + Effect.sync(() => { + receivedRuleEvidence = input.ruleEvidence; + return { score: 85, label: "Good" }; + }), + }), + ); + + await Effect.runPromise( + runInspect({ + ...baseInput, + collectScoreEvidence: (diagnostics) => { + expect(diagnostics).toEqual([lintDiagnostic]); + return [ruleEvidence]; + }, + }).pipe(Effect.provide(layersOf({ diagnostics: [lintDiagnostic], scoreLayer }))), + ); + + expect(receivedRuleEvidence).toEqual([ruleEvidence]); + }); + it("adds local authenticated GitHub viewer permission to score metadata", async () => { const output = await Effect.runPromise( runInspect({ ...baseInput, resolveLocalGithubViewerPermission: true }).pipe( diff --git a/packages/react-doctor/README.md b/packages/react-doctor/README.md index ca73d3cc55..6d96d7fb5d 100644 --- a/packages/react-doctor/README.md +++ b/packages/react-doctor/README.md @@ -65,7 +65,7 @@ We collect: - Invocation: which command, package manager, and run context (whether it's local vs. CI vs. coding agent) - Project shape: framework, React version, TypeScript, and project size - Rules fired: rule names and counts (e.g. `react-doctor/no-array-index-as-key`) -- Rule evidence: minimized token patterns from source spans that trigger diagnostics. React Doctor replaces identifier names and literal contents, removes comments and file paths, and limits the number and size of patterns. It does not collect complete source files. We use these patterns to fix false positives, which are incorrect diagnostics, and false negatives, which are issues that rules miss +- Rule evidence: minimized token patterns from source spans that trigger diagnostics. React Doctor replaces identifier names and literal contents, removes comments and file paths, and limits the number and size of patterns. It does not collect complete source files. Patterns are sent with score data and may be stored with repository and commit details when available. We use them to fix false positives, which are incorrect diagnostics, and false negatives, which are issues that rules miss - De-minified React Doctor CLI stack traces To disable telemetry for a run: `npx react-doctor@latest --no-telemetry` diff --git a/packages/react-doctor/src/cli/utils/collect-rule-evidence.ts b/packages/react-doctor/src/cli/utils/collect-rule-evidence.ts index cbf44f5a84..7c38ca667f 100644 --- a/packages/react-doctor/src/cli/utils/collect-rule-evidence.ts +++ b/packages/react-doctor/src/cli/utils/collect-rule-evidence.ts @@ -1,29 +1,19 @@ import { getDiagnosticRuleIdentity } from "@react-doctor/core"; -import type { Diagnostic } from "@react-doctor/core"; +import type { Diagnostic, ScoreRuleEvidence } from "@react-doctor/core"; import { anonymizeDiagnosticEvidence } from "./anonymize-diagnostic-evidence.js"; import { RULE_EVIDENCE_MAX_DIAGNOSTIC_COUNT, RULE_EVIDENCE_MAX_PER_RULE_COUNT, + RULE_EVIDENCE_SCHEMA_VERSION, } from "./constants.js"; import { createDiagnosticEvidenceReader } from "./read-diagnostic-evidence.js"; -export interface RuleEvidenceRecord { - readonly category: string; - readonly fileContext: string; - readonly pattern: string; - readonly plugin: string; - readonly rule: string; - readonly severity: string; - readonly tokenCount: number; - readonly truncated: boolean; -} - export const collectRuleEvidence = ( directory: string, diagnostics: ReadonlyArray, -): RuleEvidenceRecord[] => { +): ScoreRuleEvidence[] => { const readEvidence = createDiagnosticEvidenceReader(directory); - const evidenceRecords: RuleEvidenceRecord[] = []; + const evidenceRecords: ScoreRuleEvidence[] = []; const patternKeys = new Set(); const ruleCounts = new Map(); @@ -41,6 +31,7 @@ export const collectRuleEvidence = ( patternKeys.add(patternKey); ruleCounts.set(ruleKey, (ruleCounts.get(ruleKey) ?? 0) + 1); evidenceRecords.push({ + schemaVersion: RULE_EVIDENCE_SCHEMA_VERSION, category, fileContext: diagnostic.fileContext ?? "production", pattern: anonymizedEvidence.pattern, diff --git a/packages/react-doctor/src/cli/utils/collect-score-evidence.ts b/packages/react-doctor/src/cli/utils/collect-score-evidence.ts new file mode 100644 index 0000000000..0b9a23dfec --- /dev/null +++ b/packages/react-doctor/src/cli/utils/collect-score-evidence.ts @@ -0,0 +1,28 @@ +import type { Diagnostic, ScoreRuleEvidence } from "@react-doctor/core"; +import { collectRuleEvidence } from "./collect-rule-evidence.js"; +import { METRIC } from "./constants.js"; +import { recordCount } from "./record-metric.js"; + +export interface CollectScoreEvidenceInput { + readonly diagnostics: ReadonlyArray; + readonly directory: string; +} + +export const collectScoreEvidence = ( + input: CollectScoreEvidenceInput, +): ReadonlyArray => { + try { + const evidenceRecords = collectRuleEvidence(input.directory, input.diagnostics); + for (const evidence of evidenceRecords) { + recordCount(METRIC.ruleEvidenceCollected, 1, { + rule: evidence.rule, + plugin: evidence.plugin, + category: evidence.category, + severity: evidence.severity, + }); + } + return evidenceRecords; + } catch { + return []; + } +}; diff --git a/packages/react-doctor/src/cli/utils/record-rule-evidence.ts b/packages/react-doctor/src/cli/utils/record-rule-evidence.ts deleted file mode 100644 index e663bfeb5f..0000000000 --- a/packages/react-doctor/src/cli/utils/record-rule-evidence.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { Diagnostic } from "@react-doctor/core"; -import { collectRuleEvidence } from "./collect-rule-evidence.js"; -import { METRIC, NANOSECONDS_PER_MILLISECOND, RULE_EVIDENCE_SCHEMA_VERSION } from "./constants.js"; -import { recordCount } from "./record-metric.js"; -import type { RunRootSpan } from "./with-run-span.js"; - -export interface RecordRuleEvidenceInput { - readonly diagnostics: ReadonlyArray; - readonly directory: string; - readonly rootSpan: RunRootSpan; -} - -export const recordRuleEvidence = (input: RecordRuleEvidenceInput): void => { - if (input.rootSpan === undefined) return; - try { - for (const evidence of collectRuleEvidence(input.directory, input.diagnostics)) { - input.rootSpan.event("rule.evidence", BigInt(Date.now()) * NANOSECONDS_PER_MILLISECOND, { - "evidence.schemaVersion": RULE_EVIDENCE_SCHEMA_VERSION, - "evidence.outcome": "diagnostic", - "evidence.pattern": evidence.pattern, - "evidence.tokenCount": evidence.tokenCount, - "evidence.truncated": evidence.truncated, - "evidence.fileContext": evidence.fileContext, - rule: evidence.rule, - plugin: evidence.plugin, - category: evidence.category, - severity: evidence.severity, - }); - recordCount(METRIC.ruleEvidenceCollected, 1, { - rule: evidence.rule, - plugin: evidence.plugin, - category: evidence.category, - severity: evidence.severity, - }); - } - } catch {} -}; diff --git a/packages/react-doctor/src/cli/utils/render-and-record-scan.ts b/packages/react-doctor/src/cli/utils/render-and-record-scan.ts index 09f1f1e612..730be82f6a 100644 --- a/packages/react-doctor/src/cli/utils/render-and-record-scan.ts +++ b/packages/react-doctor/src/cli/utils/render-and-record-scan.ts @@ -11,7 +11,6 @@ import { type InspectExecutionCacheStats, } from "./finalize-inspect-result.js"; import { makeNoopConsole } from "./noop-console.js"; -import { recordRuleEvidence } from "./record-rule-evidence.js"; import { recordScanMetrics } from "./record-scan-metrics.js"; import { resolveWorkerTelemetry } from "./resolve-worker-telemetry.js"; import type { CachedScanPayload } from "./scan-result-cache-payload.js"; @@ -108,11 +107,6 @@ export const renderAndRecordScan = async ( userConfig: input.payload.userConfig, suppressedRuleCounts: input.payload.suppressedRuleCounts ?? [], }); - recordRuleEvidence({ - diagnostics: result.diagnostics, - directory: input.directory, - rootSpan: input.rootSpan, - }); recordRunEvent(input.rootSpan, { ...buildRunEventConfig(input.options, input.payload.userConfig, resolvedWorkerCount), result, diff --git a/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts b/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts index eca4f2e6b9..8d5e2ed411 100644 --- a/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts +++ b/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts @@ -11,6 +11,7 @@ const TELEMETRY_DISCLOSURE_GATE: Gate = { export const TELEMETRY_DISCLOSURE_LINES = [ "React Doctor telemetry is on by default.", "Telemetry includes usage data and minimized, de-identified code patterns.", + "Patterns are sent with score data, which may include repository and commit details.", "We use these patterns to fix false positives, which are incorrect diagnostics.", "We also use them to fix false negatives, which are issues that rules miss.", "React Doctor does not collect complete source files.", diff --git a/packages/react-doctor/src/inspect.ts b/packages/react-doctor/src/inspect.ts index 70cbe6fb5c..a6b27d190d 100644 --- a/packages/react-doctor/src/inspect.ts +++ b/packages/react-doctor/src/inspect.ts @@ -32,6 +32,7 @@ import { makeNoopConsole } from "./cli/utils/noop-console.js"; import { resolveOxlintNode } from "./cli/utils/resolve-oxlint-node.js"; import { resolveInspectOptions } from "./cli/utils/resolve-inspect-options.js"; import { buildRunEventConfig } from "./cli/utils/render-and-record-scan.js"; +import { collectScoreEvidence } from "./cli/utils/collect-score-evidence.js"; import { countIncompleteLintFiles, runBaselineComparison, @@ -276,6 +277,10 @@ const runInspectWithRuntime = async ( isCi: options.isCi, doctorVersion: VERSION, runId: getRunId(), + collectScoreEvidence: + rootSpan === undefined + ? undefined + : (diagnostics) => collectScoreEvidence({ directory, diagnostics }), resolveLocalGithubViewerPermission: !options.noScore, suppressScanSummary: options.suppressRendering, supplyChainManifestChanged: options.supplyChainManifestChanged, diff --git a/packages/react-doctor/tests/collect-rule-evidence.test.ts b/packages/react-doctor/tests/collect-rule-evidence.test.ts index 91e4bc1285..7c526aab67 100644 --- a/packages/react-doctor/tests/collect-rule-evidence.test.ts +++ b/packages/react-doctor/tests/collect-rule-evidence.test.ts @@ -42,6 +42,7 @@ describe("collectRuleEvidence", () => { it("collects bounded source patterns without file paths, identifiers, or literals", () => { const [record] = collectRuleEvidence(directory, [buildDiagnostic()]); + expect(record?.schemaVersion).toBe(1); expect(record?.rule).toBe("react-doctor/no-direct-set-state-in-use-effect"); expect(record?.pattern).toContain("identifier_1"); expect(record?.pattern).not.toContain("example.tsx"); From 265e8f4736307dd4d1a488fe7df10b7512362bf1 Mon Sep 17 00:00:00 2001 From: Aiden Bai Date: Tue, 11 Aug 2026 00:56:02 +0000 Subject: [PATCH 4/5] chore(skills): make reproer provider neutral --- .agents/skills/reproer/agents/openai.yaml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 .agents/skills/reproer/agents/openai.yaml diff --git a/.agents/skills/reproer/agents/openai.yaml b/.agents/skills/reproer/agents/openai.yaml deleted file mode 100644 index 0790dad8dc..0000000000 --- a/.agents/skills/reproer/agents/openai.yaml +++ /dev/null @@ -1,4 +0,0 @@ -interface: - display_name: "Reproer" - short_description: "Create synthetic repros from token evidence" - default_prompt: "Use $reproer to turn this de-identified rule-evidence pattern into synthetic React Doctor fuzz cases." From 11bb29d88c2c13457acb4c1c5cea3d90d3e89d43 Mon Sep 17 00:00:00 2001 From: Aiden Bai Date: Tue, 11 Aug 2026 04:19:20 +0000 Subject: [PATCH 5/5] docs(privacy): clarify telemetry data handling --- .agents/skills/reproer/SKILL.md | 2 +- packages/react-doctor/README.md | 4 ++-- packages/react-doctor/src/cli/index.ts | 2 +- .../react-doctor/src/cli/utils/telemetry-disclosure.ts | 9 ++++++--- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.agents/skills/reproer/SKILL.md b/.agents/skills/reproer/SKILL.md index 160bfdfbe2..f435f2df44 100644 --- a/.agents/skills/reproer/SKILL.md +++ b/.agents/skills/reproer/SKILL.md @@ -1,6 +1,6 @@ --- name: reproer -description: Turn React Doctor `rule.evidence` telemetry events or de-identified token patterns into synthetic, parseable React/TypeScript repro hypotheses and fuzz cases. Use when investigating possible false positives or false negatives from an `evidence.pattern`, reconstructing a minimal rule-triggering shape, producing adversarial variants, or promoting a verified hypothesis into the React Doctor fuzz corpus. +description: Turn React Doctor `rule.evidence` telemetry events or identifier-redacted token patterns into synthetic, parseable React/TypeScript repro hypotheses and fuzz cases. Use when investigating possible false positives or false negatives from an `evidence.pattern`, reconstructing a minimal rule-triggering shape, producing adversarial variants, or promoting a verified hypothesis into the React Doctor fuzz corpus. --- # Reproer diff --git a/packages/react-doctor/README.md b/packages/react-doctor/README.md index 6d96d7fb5d..af5619252c 100644 --- a/packages/react-doctor/README.md +++ b/packages/react-doctor/README.md @@ -65,10 +65,10 @@ We collect: - Invocation: which command, package manager, and run context (whether it's local vs. CI vs. coding agent) - Project shape: framework, React version, TypeScript, and project size - Rules fired: rule names and counts (e.g. `react-doctor/no-array-index-as-key`) -- Rule evidence: minimized token patterns from source spans that trigger diagnostics. React Doctor replaces identifier names and literal contents, removes comments and file paths, and limits the number and size of patterns. It does not collect complete source files. Patterns are sent with score data and may be stored with repository and commit details when available. We use them to fix false positives, which are incorrect diagnostics, and false negatives, which are issues that rules miss +- Rule evidence: up to 24 minimized, identifier-redacted token patterns from source spans that trigger diagnostics, with no more than 3 patterns per rule and 160 tokens per pattern. Patterns contain no identifier names, literal contents, comments, or file paths. Score submissions also contain diagnostic file paths after path and secret scrubbing, line and column numbers, diagnostic messages, and help text. Submissions may include repository and commit details. The score service logs the request IP address and user agent. We use the patterns to fix false positives, which are incorrect diagnostics, and false negatives, which are issues that rules miss. React Doctor does not collect complete source files - De-minified React Doctor CLI stack traces -To disable telemetry for a run: `npx react-doctor@latest --no-telemetry` +To disable telemetry and skip the score API and share URL for a run: `npx react-doctor@latest --no-telemetry` ## Contributing diff --git a/packages/react-doctor/src/cli/index.ts b/packages/react-doctor/src/cli/index.ts index 7efa78e5cf..3acba8d81e 100644 --- a/packages/react-doctor/src/cli/index.ts +++ b/packages/react-doctor/src/cli/index.ts @@ -212,7 +212,7 @@ const program = new Command() ) .option( "--no-telemetry", - "disable all telemetry, including de-identified code patterns (also skips the score API and share URL)", + "disable all telemetry, including identifier-redacted code patterns (also skips the score API and share URL)", ) .option( "--staged", diff --git a/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts b/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts index 8d5e2ed411..fa513dfe42 100644 --- a/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts +++ b/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts @@ -10,12 +10,15 @@ const TELEMETRY_DISCLOSURE_GATE: Gate = { export const TELEMETRY_DISCLOSURE_LINES = [ "React Doctor telemetry is on by default.", - "Telemetry includes usage data and minimized, de-identified code patterns.", - "Patterns are sent with score data, which may include repository and commit details.", + "Telemetry includes usage data, crash reports, and minimized, identifier-redacted token patterns.", + "Patterns contain no identifier names, literal contents, comments, or file paths.", + "Score submissions also include scrubbed diagnostic paths, locations, messages, and help text.", + "Score data may include repository and commit details.", + "The score service logs the request IP address and user agent.", "We use these patterns to fix false positives, which are incorrect diagnostics.", "We also use them to fix false negatives, which are issues that rules miss.", "React Doctor does not collect complete source files.", - "Run with --no-telemetry to disable telemetry.", + "Run with --no-telemetry to disable telemetry and skip the score API and share URL.", ]; export interface ShowTelemetryDisclosureInput {