diff --git a/.agents/skills/reproer/SKILL.md b/.agents/skills/reproer/SKILL.md new file mode 100644 index 0000000000..f435f2df44 --- /dev/null +++ b/.agents/skills/reproer/SKILL.md @@ -0,0 +1,93 @@ +--- +name: reproer +description: Turn React Doctor `rule.evidence` telemetry events or identifier-redacted token patterns into synthetic, parseable React/TypeScript repro hypotheses and fuzz cases. Use when investigating possible false positives or false negatives from an `evidence.pattern`, reconstructing a minimal rule-triggering shape, producing adversarial variants, or promoting a verified hypothesis into the React Doctor fuzz corpus. +--- + +# Reproer + +Generate synthetic programs that could produce the supplied token pattern. Treat every result as a hypothesis, never as recovered source. + +## Protect the privacy boundary + +- Never claim to reconstruct the original code. The mapping from source to evidence is many-to-one. +- Never search GitHub, the internet, or private repositories to identify a matching source unless the user explicitly requests that separate investigation. +- Invent neutral identifiers and literal values. Do not guess names, domains, secrets, UI copy, package names, or repository identity. +- Preserve an `identifier_N` equality relationship only within one candidate. Do not correlate placeholders across events. +- Keep the telemetry event out of committed fixtures. Commit only synthetic code written from the structural hypothesis. +- Describe reconstruction confidence as structural fidelity, not source fidelity. + +## Require the useful inputs + +Collect these fields when available: + +```text +rule +evidence.pattern +evidence.fileContext +evidence.tokenCount +evidence.truncated +category +severity +``` + +Require `rule` and `evidence.pattern`. Ask for either field if missing because the rule supplies semantics that the anonymized identifiers no longer carry. + +## Build repro hypotheses + +1. Locate and read the target rule, its metadata, focused tests, and any existing fuzz fixtures. Use `rg` with the short rule ID. +2. Read `.agents/skills/fuzz/SKILL.md` and `packages/fuzz/README.md` before running the harness or modifying its corpus. +3. Decode the pattern conservatively: + - Map each `identifier_N` to one neutral role name and reuse it within the candidate. + - Replace `string_literal`, `number_literal`, `boolean_literal`, `null_literal`, `bigint_literal`, `regular_expression_literal`, and `template_literal` with harmless canonical values. + - Preserve keywords, operators, delimiters, member access, calls, and identifier-equality relationships. + - Treat `syntax_N` as unknown until checking the matching TypeScript `SyntaxKind`. + - Add the smallest component, hook, import, or function wrapper needed to parse and exercise the rule. + - When `evidence.truncated` is true, close the program into a valid minimal shape without pretending the invented suffix matches the source. +4. Produce a small hypothesis matrix: + - **Nearest shape:** retain the observed token order with only parseability scaffolding. + - **Valid-context shape:** place the syntax in the most ordinary legitimate React or TypeScript context. + - **False-positive probe:** create a valid program where the suspicious shape is intentional or safe. + - **False-negative probes:** change one dimension at a time, such as aliasing, wrappers, optional chaining, parentheses, control flow, TypeScript syntax, JSX placement, or cross-statement data flow. +5. Generate three to eight candidates. Avoid a combinatorial matrix until one candidate reaches the rule's reporting path. + +## Verify instead of guessing + +Place initial candidates in a temporary directory or an existing focused test harness. Do not add corpus files before checking the verdict. + +For every candidate, record: + +- whether it parses; +- whether the target rule fires; +- the diagnostic span and message; +- the intended semantic verdict: valid, invalid, or uncertain; +- the mutation dimension that distinguishes it from the nearest shape. + +Do not call a case a false positive or false negative from telemetry alone. Confirm the rule contract and the program's semantics first. + +Run the narrowest focused rule test. After it passes, run the targeted fuzzer: + +```sh +FUZZ_RULE= FUZZ_STRICT=1 FUZZ_ITERATIONS=500 nr fuzz +``` + +Confirm that the target rule fires at least once. A silent run validates only early exits. + +## Promote verified cases + +- For a confirmed false positive, add a minimal focused valid case and `packages/fuzz/corpus/regressions/--.tsx` with `// verdict: pass`. +- For a confirmed true positive or liveness seed, add a minimal focused invalid case and use `packages/fuzz/corpus/targets/` when the harness needs a reporting-path seed. +- For a confirmed false negative, add the missing invalid case to the focused rule tests, fix the detector when requested, and add a generator snippet only when existing pools cannot produce the weakness. +- Use stable weakness names from the fuzz skill. Never label an unverified hypothesis as a regression. +- Run the focused test, `nr -C packages/fuzz test`, and replay the target rule after changing the corpus. + +## Report the result + +Return: + +1. the normalized evidence skeleton; +2. the synthetic candidates and their structural-fidelity notes; +3. the observed rule verdict for each candidate; +4. confirmed FP, FN, or liveness findings, clearly separated from hypotheses; +5. tests, fuzz commands, seeds, and files changed. + +State explicitly that the candidates do not recover identifiers, literals, comments, paths, or repository identity. diff --git a/.changeset/bright-rules-report.md b/.changeset/bright-rules-report.md new file mode 100644 index 0000000000..ef69b5f4ff --- /dev/null +++ b/.changeset/bright-rules-report.md @@ -0,0 +1,5 @@ +--- +"react-doctor": patch +--- + +Collect minimized diagnostic source patterns with score telemetry to help fix false positives and false negatives, with score-level indexing, a one-time interactive disclosure, and a `--no-telemetry` opt-out. diff --git a/packages/core/src/request-score.ts b/packages/core/src/request-score.ts index 6051f40251..1a73caa232 100644 --- a/packages/core/src/request-score.ts +++ b/packages/core/src/request-score.ts @@ -84,6 +84,7 @@ const buildScoreRequestBody = ( return gzipSync( JSON.stringify({ diagnostics: sanitizeScoreDiagnostics(diagnostics), + ...(options.ruleEvidence !== undefined ? { ruleEvidence: options.ruleEvidence } : {}), ...buildScoreRequestMetadata(options.metadata), }), ); diff --git a/packages/core/src/run-inspect.ts b/packages/core/src/run-inspect.ts index c139403eaf..c30e04d30e 100644 --- a/packages/core/src/run-inspect.ts +++ b/packages/core/src/run-inspect.ts @@ -816,6 +816,7 @@ export const runInspect = ( diagnostics: scoreDiagnostics, isCi: input.isCi, metadata: scoreMetadata, + ruleEvidence: input.collectScoreEvidence?.(scoreDiagnostics), }); const lintPartialFailures = yield* Ref.get(partialFailuresRef); const didSecurityScanFail = yield* Ref.get(securityScanFailedRef); diff --git a/packages/core/src/services/score.ts b/packages/core/src/services/score.ts index 85f300ebd9..67e7fef82f 100644 --- a/packages/core/src/services/score.ts +++ b/packages/core/src/services/score.ts @@ -3,13 +3,19 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient"; import * as HttpClient from "effect/unstable/http/HttpClient"; -import type { Diagnostic, ScoreRequestMetadata, ScoreResult } from "../types/index.js"; +import type { + Diagnostic, + ScoreRequestMetadata, + ScoreResult, + ScoreRuleEvidence, +} from "../types/index.js"; import { requestScore } from "../request-score.js"; interface ComputeInput { readonly diagnostics: ReadonlyArray; readonly isCi?: boolean; readonly metadata?: ScoreRequestMetadata; + readonly ruleEvidence?: ReadonlyArray; } export class Score extends Context.Service< @@ -27,6 +33,7 @@ export class Score extends Context.Service< requestScore(httpClient, input.diagnostics, { isCi: input.isCi, metadata: input.metadata, + ruleEvidence: input.ruleEvidence, }), ), }); diff --git a/packages/core/src/types/index.ts b/packages/core/src/types/index.ts index 0c6c70a2f8..c7f72c6948 100644 --- a/packages/core/src/types/index.ts +++ b/packages/core/src/types/index.ts @@ -66,6 +66,7 @@ export type { PromptMultiselectChoiceState, PromptMultiselectContext } from "./p // See that file for the duplication rationale. export type { CalculateScoreOptions, + ScoreRuleEvidence, ScoreRequestMetadata, ScoreResult, RulePriority, diff --git a/packages/core/src/types/run-inspect.ts b/packages/core/src/types/run-inspect.ts index d250b12512..d277a8799d 100644 --- a/packages/core/src/types/run-inspect.ts +++ b/packages/core/src/types/run-inspect.ts @@ -4,7 +4,7 @@ import type { DiagnosticSurface, ReactDoctorConfig } from "./config.js"; import type { Diagnostic, SourceFileEntry, SuppressedRuleCount } from "./diagnostic.js"; import type { ChangedFileLineRanges } from "./inspect.js"; import type { ProjectInfo } from "./project-info.js"; -import type { ScoreRequestMetadata, ScoreResult } from "./score.js"; +import type { ScoreRequestMetadata, ScoreResult, ScoreRuleEvidence } from "./score.js"; export interface InspectInput { readonly directory: string; @@ -35,6 +35,9 @@ export interface InspectInput { readonly doctorVersion?: string; /** Random per-run id. */ readonly runId?: string; + readonly collectScoreEvidence?: ( + diagnostics: ReadonlyArray, + ) => ReadonlyArray; /** Enables best-effort authenticated local GitHub permission lookup for score metadata. */ readonly resolveLocalGithubViewerPermission?: boolean; /** diff --git a/packages/core/src/types/score.ts b/packages/core/src/types/score.ts index 09d806d2a4..e216930f1b 100644 --- a/packages/core/src/types/score.ts +++ b/packages/core/src/types/score.ts @@ -1,10 +1,24 @@ import type { ProjectInfo } from "./project-info.js"; +import type { Diagnostic, DiagnosticFileContext } from "./diagnostic.js"; export type RuleTier = "P0" | "P1" | "P2" | "P3"; export interface CalculateScoreOptions { isCi?: boolean; metadata?: ScoreRequestMetadata; + ruleEvidence?: ReadonlyArray; +} + +export interface ScoreRuleEvidence { + readonly schemaVersion: 1; + readonly category: string; + readonly fileContext: DiagnosticFileContext; + readonly pattern: string; + readonly plugin: string; + readonly rule: string; + readonly severity: Diagnostic["severity"]; + readonly tokenCount: number; + readonly truncated: boolean; } export interface ScoreRequestMetadata { diff --git a/packages/core/tests/calculate-score.test.ts b/packages/core/tests/calculate-score.test.ts index d4fd9c1dc3..8c92d512f1 100644 --- a/packages/core/tests/calculate-score.test.ts +++ b/packages/core/tests/calculate-score.test.ts @@ -82,6 +82,19 @@ describe("calculateScore", () => { githubActorAssociation: "CONTRIBUTOR", githubViewerPermission: "write", }, + ruleEvidence: [ + { + schemaVersion: 1, + category: "State & Effects", + fileContext: "production", + pattern: "identifier_1 ( )", + plugin: "react-doctor", + rule: "react-doctor/example-rule", + severity: "error", + tokenCount: 4, + truncated: false, + }, + ], }); expect(result).toEqual(apiScoreResponse); @@ -109,6 +122,13 @@ describe("calculateScore", () => { githubEventName: "pull_request", githubActorAssociation: "CONTRIBUTOR", githubViewerPermission: "write", + ruleEvidence: [ + { + schemaVersion: 1, + pattern: "identifier_1 ( )", + rule: "react-doctor/example-rule", + }, + ], }); }); diff --git a/packages/core/tests/run-inspect.test.ts b/packages/core/tests/run-inspect.test.ts index 2e0f375653..daf8067b5a 100644 --- a/packages/core/tests/run-inspect.test.ts +++ b/packages/core/tests/run-inspect.test.ts @@ -12,6 +12,7 @@ import type { Diagnostic, ProjectInfo, ReactDoctorConfig, + ScoreRuleEvidence, SourceFileEntry, } from "@react-doctor/core"; import { @@ -502,6 +503,43 @@ describe("runInspect — happy path", () => { expect(output.didDeadCodeFail).toBe(false); }); + it("attaches collected rule evidence to the score request", async () => { + const ruleEvidence: ScoreRuleEvidence = { + schemaVersion: 1, + category: "Correctness", + fileContext: "production", + pattern: "identifier_1 ( )", + plugin: "react-doctor", + rule: "react-doctor/no-derived-state", + severity: "error", + tokenCount: 4, + truncated: false, + }; + let receivedRuleEvidence: ReadonlyArray | undefined; + const scoreLayer = Layer.succeed( + Score, + Score.of({ + compute: (input) => + Effect.sync(() => { + receivedRuleEvidence = input.ruleEvidence; + return { score: 85, label: "Good" }; + }), + }), + ); + + await Effect.runPromise( + runInspect({ + ...baseInput, + collectScoreEvidence: (diagnostics) => { + expect(diagnostics).toEqual([lintDiagnostic]); + return [ruleEvidence]; + }, + }).pipe(Effect.provide(layersOf({ diagnostics: [lintDiagnostic], scoreLayer }))), + ); + + expect(receivedRuleEvidence).toEqual([ruleEvidence]); + }); + it("adds local authenticated GitHub viewer permission to score metadata", async () => { const output = await Effect.runPromise( runInspect({ ...baseInput, resolveLocalGithubViewerPermission: true }).pipe( diff --git a/packages/react-doctor/README.md b/packages/react-doctor/README.md index cadb743b63..93408db4f7 100644 --- a/packages/react-doctor/README.md +++ b/packages/react-doctor/README.md @@ -84,17 +84,18 @@ URLs, source paths, and React profiling details. Treat it as sensitive applicati ## Telemetry -The CLI reports crashes, basic run traces, and anonymous usage counters to [Sentry](https://sentry.io/) to help us fix bugs and prioritize work. +Telemetry is on by default. The CLI sends crash reports to [Sentry](https://sentry.io/) and usage telemetry to [Axiom](https://axiom.co/). We use this data to operate React Doctor and improve its diagnostic rules. We collect: - Environment: CLI version, platform, Node version - Invocation: which command, package manager, and run context (whether it's local vs. CI vs. coding agent) -- Project shape: framework, React version, TypeScript, project size (NO file contents) -- Rules fired: rule names and counts only (e.g. `react-doctor/no-array-index-as-key`) (NO code or specific findings) +- Project shape: framework, React version, TypeScript, and project size +- Rules fired: rule names and counts (e.g. `react-doctor/no-array-index-as-key`) +- Rule evidence: up to 24 minimized, identifier-redacted token patterns from source spans that trigger diagnostics, with no more than 3 patterns per rule and 160 tokens per pattern. Patterns contain no identifier names, literal contents, comments, or file paths. Score submissions also contain diagnostic file paths after path and secret scrubbing, line and column numbers, diagnostic messages, and help text. Submissions may include repository and commit details. The score service logs the request IP address and user agent. We use the patterns to fix false positives, which are incorrect diagnostics, and false negatives, which are issues that rules miss. React Doctor does not collect complete source files - De-minified React Doctor CLI stack traces -To opt out, run: `npx react-doctor@latest --no-telemetry` +To disable telemetry and skip the score API and share URL for a run: `npx react-doctor@latest --no-telemetry` ## Contributing diff --git a/packages/react-doctor/src/cli/commands/scan.ts b/packages/react-doctor/src/cli/commands/scan.ts index c28385255e..73854ea59d 100644 --- a/packages/react-doctor/src/cli/commands/scan.ts +++ b/packages/react-doctor/src/cli/commands/scan.ts @@ -9,6 +9,7 @@ import { resolveCliInspectOptions } from "../utils/resolve-cli-inspect-options.j import { resolveTuiEnvironment } from "../utils/resolve-tui-environment.js"; import { warnDeprecatedDiff } from "../utils/resolve-scope.js"; import { shouldUseTui } from "../utils/should-use-tui.js"; +import { showTelemetryDisclosureIfNeeded } from "../utils/telemetry-disclosure.js"; import { validateModeFlags } from "../utils/validate-mode-flags.js"; import { warnDeprecatedFailOn } from "../utils/warn-deprecated-fail-on.js"; @@ -27,6 +28,16 @@ export const runScanCommand = async (input: RunScanCommandInput): Promise if (!shouldUseTui(tuiEnvironment)) { await inspectAction(input.directory, input.flags, input.invocationCommand); + showTelemetryDisclosureIfNeeded({ + isInteractive: + !tuiEnvironment.isNonInteractiveEnvironment && + tuiEnvironment.stdinIsTty && + tuiEnvironment.outputIsTty && + input.flags.json !== true && + input.flags.jsonCompact !== true && + input.flags.jsonOut === undefined && + input.flags.score !== true, + }); return; } @@ -47,4 +58,5 @@ export const runScanCommand = async (input: RunScanCommandInput): Promise flags: input.flags, }); if (shouldFail) process.exitCode = 1; + showTelemetryDisclosureIfNeeded({ isInteractive: true }); }; diff --git a/packages/react-doctor/src/cli/index.ts b/packages/react-doctor/src/cli/index.ts index f6c5dd6758..1af0b4075e 100644 --- a/packages/react-doctor/src/cli/index.ts +++ b/packages/react-doctor/src/cli/index.ts @@ -237,7 +237,7 @@ const program = new Command() "scan source files listed in a newline-delimited changed-files file", ).hideHelp(), ) - .option("--no-score", "skip the score API, the share URL, and crash reporting") + .option("--no-score", "skip the score API, the share URL, and all telemetry") .addOption( new Option( "--category ", @@ -246,7 +246,7 @@ const program = new Command() ) .option( "--no-telemetry", - "alias for --no-score (skip the score API, share URL, and crash reporting)", + "disable all telemetry, including identifier-redacted code patterns (also skips the score API and share URL)", ) .option( "--staged", @@ -521,7 +521,7 @@ program .addOption(new Option("--no-dead-code").hideHelp()) .option("--no-supply-chain", "skip the dependency supply-chain scan") .option("--score", "only print the numeric score (for scripts and CI)") - .option("--no-score", "skip the score API, the share URL, and crash reporting") + .option("--no-score", "skip the score API, the share URL, and all telemetry") .option("--no-cache", "disable all scan caches for this run") .option("--max-duration ", MAX_DURATION_OPTION_DESCRIPTION) .option("-p, --project ", "scan specific workspace projects (comma-separated, or *)") diff --git a/packages/react-doctor/src/cli/utils/anonymize-diagnostic-evidence.ts b/packages/react-doctor/src/cli/utils/anonymize-diagnostic-evidence.ts new file mode 100644 index 0000000000..2033082777 --- /dev/null +++ b/packages/react-doctor/src/cli/utils/anonymize-diagnostic-evidence.ts @@ -0,0 +1,65 @@ +import ts from "typescript"; +import { RULE_EVIDENCE_MAX_TOKEN_COUNT } from "./constants.js"; + +export interface AnonymizedDiagnosticEvidence { + readonly pattern: string; + readonly tokenCount: number; + readonly truncated: boolean; +} + +export const anonymizeDiagnosticEvidence = (evidence: string): AnonymizedDiagnosticEvidence => { + const scanner = ts.createScanner(ts.ScriptTarget.Latest, true, ts.LanguageVariant.JSX, evidence); + const identifierIndexes = new Map(); + const tokens: string[] = []; + let tokenCount = 0; + let tokenKind = scanner.scan(); + + while (tokenKind !== ts.SyntaxKind.EndOfFileToken) { + tokenCount += 1; + if (tokens.length < RULE_EVIDENCE_MAX_TOKEN_COUNT) { + let token: string; + if (tokenKind === ts.SyntaxKind.Identifier || tokenKind === ts.SyntaxKind.PrivateIdentifier) { + const identifier = scanner.getTokenText(); + const identifierIndex = identifierIndexes.get(identifier) ?? identifierIndexes.size + 1; + identifierIndexes.set(identifier, identifierIndex); + token = `identifier_${identifierIndex}`; + } else if ( + tokenKind === ts.SyntaxKind.StringLiteral || + tokenKind === ts.SyntaxKind.JsxText || + tokenKind === ts.SyntaxKind.JsxTextAllWhiteSpaces + ) { + token = "string_literal"; + } else if (tokenKind === ts.SyntaxKind.NumericLiteral) { + token = "number_literal"; + } else if ( + tokenKind === ts.SyntaxKind.TrueKeyword || + tokenKind === ts.SyntaxKind.FalseKeyword + ) { + token = "boolean_literal"; + } else if (tokenKind === ts.SyntaxKind.NullKeyword) { + token = "null_literal"; + } else if (tokenKind === ts.SyntaxKind.BigIntLiteral) { + token = "bigint_literal"; + } else if (tokenKind === ts.SyntaxKind.RegularExpressionLiteral) { + token = "regular_expression_literal"; + } else if ( + tokenKind === ts.SyntaxKind.NoSubstitutionTemplateLiteral || + tokenKind === ts.SyntaxKind.TemplateHead || + tokenKind === ts.SyntaxKind.TemplateMiddle || + tokenKind === ts.SyntaxKind.TemplateTail + ) { + token = "template_literal"; + } else { + token = ts.tokenToString(tokenKind) ?? `syntax_${tokenKind}`; + } + tokens.push(token); + } + tokenKind = scanner.scan(); + } + + return { + pattern: tokens.join(" "), + tokenCount, + truncated: tokenCount > RULE_EVIDENCE_MAX_TOKEN_COUNT, + }; +}; diff --git a/packages/react-doctor/src/cli/utils/cli-state-store.ts b/packages/react-doctor/src/cli/utils/cli-state-store.ts index d21500dd54..0c670b2355 100644 --- a/packages/react-doctor/src/cli/utils/cli-state-store.ts +++ b/packages/react-doctor/src/cli/utils/cli-state-store.ts @@ -37,11 +37,13 @@ export const INITIAL_LIFECYCLE_VERSION = 1; // surface kind scope id / migration id wired in // ──────────────────── ───────── ─────── ────────────────── ──────────────────────── // first-run onboarding gate global onboarding onboarding-state.ts +// telemetry disclosure gate global telemetry-disclosure telemetry-disclosure.ts // "add to CI?" pitch gate project ci-pitch ci-prompt-decision.ts // @v1 → @v2 offer gate project action-upgrade-v2 action-upgrade-prompt.ts // agent install hint gate project setup-hint prompt-install-setup.ts // config json → ts migration project config-json-to-ts cli-migrations.ts export const ONBOARDING_EVENT = "onboarding"; +export const TELEMETRY_DISCLOSURE_EVENT = "telemetry-disclosure"; export const CI_PITCH_EVENT = "ci-pitch"; export const ACTION_UPGRADE_EVENT = "action-upgrade-v2"; export const SETUP_HINT_EVENT = "setup-hint"; diff --git a/packages/react-doctor/src/cli/utils/collect-rule-evidence.ts b/packages/react-doctor/src/cli/utils/collect-rule-evidence.ts new file mode 100644 index 0000000000..7c38ca667f --- /dev/null +++ b/packages/react-doctor/src/cli/utils/collect-rule-evidence.ts @@ -0,0 +1,47 @@ +import { getDiagnosticRuleIdentity } from "@react-doctor/core"; +import type { Diagnostic, ScoreRuleEvidence } from "@react-doctor/core"; +import { anonymizeDiagnosticEvidence } from "./anonymize-diagnostic-evidence.js"; +import { + RULE_EVIDENCE_MAX_DIAGNOSTIC_COUNT, + RULE_EVIDENCE_MAX_PER_RULE_COUNT, + RULE_EVIDENCE_SCHEMA_VERSION, +} from "./constants.js"; +import { createDiagnosticEvidenceReader } from "./read-diagnostic-evidence.js"; + +export const collectRuleEvidence = ( + directory: string, + diagnostics: ReadonlyArray, +): ScoreRuleEvidence[] => { + const readEvidence = createDiagnosticEvidenceReader(directory); + const evidenceRecords: ScoreRuleEvidence[] = []; + const patternKeys = new Set(); + const ruleCounts = new Map(); + + for (const diagnostic of diagnostics) { + if (evidenceRecords.length >= RULE_EVIDENCE_MAX_DIAGNOSTIC_COUNT) break; + if (diagnostic.plugin !== "react-doctor") continue; + const { ruleKey, category } = getDiagnosticRuleIdentity(diagnostic); + if ((ruleCounts.get(ruleKey) ?? 0) >= RULE_EVIDENCE_MAX_PER_RULE_COUNT) continue; + const evidence = readEvidence(diagnostic); + if (evidence === null) continue; + const anonymizedEvidence = anonymizeDiagnosticEvidence(evidence); + if (anonymizedEvidence.pattern === "") continue; + const patternKey = `${ruleKey}\0${anonymizedEvidence.pattern}`; + if (patternKeys.has(patternKey)) continue; + patternKeys.add(patternKey); + ruleCounts.set(ruleKey, (ruleCounts.get(ruleKey) ?? 0) + 1); + evidenceRecords.push({ + schemaVersion: RULE_EVIDENCE_SCHEMA_VERSION, + category, + fileContext: diagnostic.fileContext ?? "production", + pattern: anonymizedEvidence.pattern, + plugin: diagnostic.plugin, + rule: ruleKey, + severity: diagnostic.severity, + tokenCount: anonymizedEvidence.tokenCount, + truncated: anonymizedEvidence.truncated, + }); + } + + return evidenceRecords; +}; diff --git a/packages/react-doctor/src/cli/utils/collect-score-evidence.ts b/packages/react-doctor/src/cli/utils/collect-score-evidence.ts new file mode 100644 index 0000000000..0b9a23dfec --- /dev/null +++ b/packages/react-doctor/src/cli/utils/collect-score-evidence.ts @@ -0,0 +1,28 @@ +import type { Diagnostic, ScoreRuleEvidence } from "@react-doctor/core"; +import { collectRuleEvidence } from "./collect-rule-evidence.js"; +import { METRIC } from "./constants.js"; +import { recordCount } from "./record-metric.js"; + +export interface CollectScoreEvidenceInput { + readonly diagnostics: ReadonlyArray; + readonly directory: string; +} + +export const collectScoreEvidence = ( + input: CollectScoreEvidenceInput, +): ReadonlyArray => { + try { + const evidenceRecords = collectRuleEvidence(input.directory, input.diagnostics); + for (const evidence of evidenceRecords) { + recordCount(METRIC.ruleEvidenceCollected, 1, { + rule: evidence.rule, + plugin: evidence.plugin, + category: evidence.category, + severity: evidence.severity, + }); + } + return evidenceRecords; + } catch { + return []; + } +}; diff --git a/packages/react-doctor/src/cli/utils/constants.ts b/packages/react-doctor/src/cli/utils/constants.ts index b2aa358400..d98daf513f 100644 --- a/packages/react-doctor/src/cli/utils/constants.ts +++ b/packages/react-doctor/src/cli/utils/constants.ts @@ -218,6 +218,11 @@ export const AXIOM_INGEST_TOKEN = "xaat-31b59107-855d-4917-8fab-6dc29fb459ce"; // Effect span clocks are epoch nanoseconds; `Date.now()` is milliseconds. export const NANOSECONDS_PER_MILLISECOND = 1_000_000n; +export const RULE_EVIDENCE_SCHEMA_VERSION = 1; +export const RULE_EVIDENCE_MAX_DIAGNOSTIC_COUNT = 24; +export const RULE_EVIDENCE_MAX_PER_RULE_COUNT = 3; +export const RULE_EVIDENCE_MAX_TOKEN_COUNT = 160; + export const AXIOM_TRACES_DATASET = "react-doctor"; export const AXIOM_METRICS_DATASET = "react-doctor-metrics"; @@ -287,6 +292,7 @@ export const METRIC = { scanNoReactDetected: "scan.no_react_detected", baselineDegraded: "baseline.degraded", ruleFired: "rule.fired", + ruleEvidenceCollected: "rule.evidence_collected", // Rule-rejection telemetry, both keyed by `rule` + `source` attributes: // `rule.disabled` counts one per scan per config-off rule (`rules: "off"` / // `ignore.rules` — the former never fires, so this is its only signal); diff --git a/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts b/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts new file mode 100644 index 0000000000..fa513dfe42 --- /dev/null +++ b/packages/react-doctor/src/cli/utils/telemetry-disclosure.ts @@ -0,0 +1,40 @@ +import { type CliStateOptions, TELEMETRY_DISCLOSURE_EVENT } from "./cli-state-store.js"; +import { type Gate, isGatePending, recordGate } from "./cli-lifecycle.js"; +import { cliLogger } from "./cli-logger.js"; +import { isTelemetryEnabled } from "./is-telemetry-enabled.js"; + +const TELEMETRY_DISCLOSURE_GATE: Gate = { + id: TELEMETRY_DISCLOSURE_EVENT, + scope: "global", +}; + +export const TELEMETRY_DISCLOSURE_LINES = [ + "React Doctor telemetry is on by default.", + "Telemetry includes usage data, crash reports, and minimized, identifier-redacted token patterns.", + "Patterns contain no identifier names, literal contents, comments, or file paths.", + "Score submissions also include scrubbed diagnostic paths, locations, messages, and help text.", + "Score data may include repository and commit details.", + "The score service logs the request IP address and user agent.", + "We use these patterns to fix false positives, which are incorrect diagnostics.", + "We also use them to fix false negatives, which are issues that rules miss.", + "React Doctor does not collect complete source files.", + "Run with --no-telemetry to disable telemetry and skip the score API and share URL.", +]; + +export interface ShowTelemetryDisclosureInput { + readonly isInteractive: boolean; + readonly store?: CliStateOptions; + readonly telemetryEnabled?: boolean; + readonly writeLine?: (line: string) => void; +} + +export const showTelemetryDisclosureIfNeeded = (input: ShowTelemetryDisclosureInput): boolean => { + if (!input.isInteractive || !(input.telemetryEnabled ?? isTelemetryEnabled())) return false; + if (!isGatePending(TELEMETRY_DISCLOSURE_GATE, {}, input.store)) return false; + const writeLine = input.writeLine ?? cliLogger.log; + writeLine(""); + for (const line of TELEMETRY_DISCLOSURE_LINES) writeLine(line); + writeLine(""); + recordGate(TELEMETRY_DISCLOSURE_GATE, { outcome: "seen" }, input.store); + return true; +}; diff --git a/packages/react-doctor/src/inspect.ts b/packages/react-doctor/src/inspect.ts index 6fde938673..e0d7c47023 100644 --- a/packages/react-doctor/src/inspect.ts +++ b/packages/react-doctor/src/inspect.ts @@ -32,6 +32,7 @@ import { makeNoopConsole } from "./cli/utils/noop-console.js"; import { resolveOxlintNode } from "./cli/utils/resolve-oxlint-node.js"; import { resolveInspectOptions } from "./cli/utils/resolve-inspect-options.js"; import { buildRunEventConfig } from "./cli/utils/render-and-record-scan.js"; +import { collectScoreEvidence } from "./cli/utils/collect-score-evidence.js"; import { countIncompleteLintFiles, runBaselineComparison, @@ -277,6 +278,10 @@ const runInspectWithRuntime = async ( isCi: options.isCi, doctorVersion: VERSION, runId: getRunId(), + collectScoreEvidence: + rootSpan === undefined || options.noScore + ? undefined + : (diagnostics) => collectScoreEvidence({ directory, diagnostics }), resolveLocalGithubViewerPermission: !options.noScore, suppressScanSummary: options.suppressRendering, supplyChainManifestChanged: options.supplyChainManifestChanged, diff --git a/packages/react-doctor/tests/anonymize-diagnostic-evidence.test.ts b/packages/react-doctor/tests/anonymize-diagnostic-evidence.test.ts new file mode 100644 index 0000000000..1c5405be96 --- /dev/null +++ b/packages/react-doctor/tests/anonymize-diagnostic-evidence.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vite-plus/test"; +import { anonymizeDiagnosticEvidence } from "../src/cli/utils/anonymize-diagnostic-evidence.js"; +import { RULE_EVIDENCE_MAX_TOKEN_COUNT } from "../src/cli/utils/constants.js"; + +describe("anonymizeDiagnosticEvidence", () => { + it("preserves syntax and binding relationships without source names or literal contents", () => { + const evidence = ` + // Customer-specific behavior + const customerEmail = "person@example.com"; + useEffect(() => setCustomer(customerEmail), [customerEmail]); + `; + + const result = anonymizeDiagnosticEvidence(evidence); + + expect(result.pattern).toContain("const identifier_1 = string_literal ;"); + expect(result.pattern).toContain("identifier_2 ( ( ) => identifier_3 ( identifier_1 )"); + expect(result.pattern).not.toContain("Customer-specific"); + expect(result.pattern).not.toContain("customerEmail"); + expect(result.pattern).not.toContain("person@example.com"); + expect(result.truncated).toBe(false); + }); + + it("uses stable placeholders for repeated identifiers", () => { + const result = anonymizeDiagnosticEvidence("value + value + other"); + + expect(result.pattern).toBe("identifier_1 + identifier_1 + identifier_2"); + }); + + it("removes primitive literal values", () => { + const result = anonymizeDiagnosticEvidence("[0, 1, 42, true, false, null]"); + + expect(result.pattern).toBe( + "[ number_literal , number_literal , number_literal , boolean_literal , boolean_literal , null_literal ]", + ); + }); + + it("bounds the exported pattern while retaining the original token count", () => { + const evidence = Array.from( + { length: RULE_EVIDENCE_MAX_TOKEN_COUNT + 10 }, + (_unused, index) => `identifier${index};`, + ).join(" "); + + const result = anonymizeDiagnosticEvidence(evidence); + + expect(result.tokenCount).toBeGreaterThan(RULE_EVIDENCE_MAX_TOKEN_COUNT); + expect(result.pattern.split(" ")).toHaveLength(RULE_EVIDENCE_MAX_TOKEN_COUNT); + expect(result.truncated).toBe(true); + }); +}); diff --git a/packages/react-doctor/tests/collect-rule-evidence.test.ts b/packages/react-doctor/tests/collect-rule-evidence.test.ts new file mode 100644 index 0000000000..7c526aab67 --- /dev/null +++ b/packages/react-doctor/tests/collect-rule-evidence.test.ts @@ -0,0 +1,77 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { Diagnostic } from "@react-doctor/core"; +import { afterEach, beforeEach, describe, expect, it } from "vite-plus/test"; +import { collectRuleEvidence } from "../src/cli/utils/collect-rule-evidence.js"; + +const buildDiagnostic = (overrides: Partial = {}): Diagnostic => ({ + filePath: "src/example.tsx", + plugin: "react-doctor", + rule: "no-direct-set-state-in-use-effect", + severity: "warning", + message: "State update in an effect", + help: "Derive the value while rendering", + line: 2, + column: 1, + category: "State & Effects", + ...overrides, +}); + +describe("collectRuleEvidence", () => { + let directory: string; + + beforeEach(() => { + directory = fs.mkdtempSync(path.join(os.tmpdir(), "react-doctor-rule-evidence-")); + fs.mkdirSync(path.join(directory, "src")); + fs.writeFileSync( + path.join(directory, "src/example.tsx"), + [ + "const customerName = 'private customer';", + "useEffect(() => setName(customerName), [customerName]);", + "const accountSecret = 'low-entropy-secret';", + "useEffect(() => setAccount({ value: accountSecret }), [accountSecret]);", + ].join("\n"), + ); + }); + + afterEach(() => { + fs.rmSync(directory, { recursive: true, force: true }); + }); + + it("collects bounded source patterns without file paths, identifiers, or literals", () => { + const [record] = collectRuleEvidence(directory, [buildDiagnostic()]); + + expect(record?.schemaVersion).toBe(1); + expect(record?.rule).toBe("react-doctor/no-direct-set-state-in-use-effect"); + expect(record?.pattern).toContain("identifier_1"); + expect(record?.pattern).not.toContain("example.tsx"); + expect(record?.pattern).not.toContain("customerName"); + expect(record?.pattern).not.toContain("private customer"); + }); + + it("deduplicates identical evidence for the same rule", () => { + const records = collectRuleEvidence(directory, [buildDiagnostic(), buildDiagnostic()]); + + expect(records).toHaveLength(1); + }); + + it("keeps distinct evidence without exposing its source text", () => { + const records = collectRuleEvidence(directory, [ + buildDiagnostic(), + buildDiagnostic({ line: 4 }), + ]); + + expect(records).toHaveLength(2); + expect(records[1]?.pattern).not.toContain("accountSecret"); + expect(records[1]?.pattern).not.toContain("low-entropy-secret"); + }); + + it("ignores diagnostics from third-party plugins", () => { + const records = collectRuleEvidence(directory, [ + buildDiagnostic({ plugin: "private-company-plugin" }), + ]); + + expect(records).toEqual([]); + }); +}); diff --git a/packages/react-doctor/tests/run-scan-command.test.ts b/packages/react-doctor/tests/run-scan-command.test.ts index 21a7bbddd1..91dc29b308 100644 --- a/packages/react-doctor/tests/run-scan-command.test.ts +++ b/packages/react-doctor/tests/run-scan-command.test.ts @@ -29,6 +29,10 @@ vi.mock("../src/cli/utils/should-use-tui.js", () => ({ shouldUseTui: vi.fn(() => true), })); +vi.mock("../src/cli/utils/telemetry-disclosure.js", () => ({ + showTelemetryDisclosureIfNeeded: vi.fn(), +})); + vi.mock("../src/cli/utils/resolve-scope.js", () => ({ warnDeprecatedDiff: vi.fn(), })); @@ -61,6 +65,7 @@ import { recordCount } from "../src/cli/utils/record-metric.js"; import { resolveCliInspectOptions } from "../src/cli/utils/resolve-cli-inspect-options.js"; import { warnDeprecatedDiff } from "../src/cli/utils/resolve-scope.js"; import { shouldUseTui } from "../src/cli/utils/should-use-tui.js"; +import { showTelemetryDisclosureIfNeeded } from "../src/cli/utils/telemetry-disclosure.js"; import { validateModeFlags } from "../src/cli/utils/validate-mode-flags.js"; import { warnDeprecatedFailOn } from "../src/cli/utils/warn-deprecated-fail-on.js"; @@ -117,6 +122,7 @@ describe("runScanCommand", () => { expect(warnDeprecatedFailOn).toHaveBeenCalledWith(flags, null); expect(warnDeprecatedDiff).toHaveBeenCalledWith(flags, null); expect(inspectAction).not.toHaveBeenCalled(); + expect(showTelemetryDisclosureIfNeeded).toHaveBeenCalledWith({ isInteractive: true }); }); it("uses headless output when the TUI gate rejects the environment or flags", async () => { @@ -129,6 +135,7 @@ describe("runScanCommand", () => { expect(runScanApp).not.toHaveBeenCalled(); expect(runProjectMigrations).not.toHaveBeenCalled(); expect(recordCount).not.toHaveBeenCalled(); + expect(showTelemetryDisclosureIfNeeded).toHaveBeenCalledWith({ isInteractive: false }); }); it("preserves the TUI scan exit code", async () => { diff --git a/packages/react-doctor/tests/score-evidence-opt-out.test.ts b/packages/react-doctor/tests/score-evidence-opt-out.test.ts new file mode 100644 index 0000000000..6d22e7c366 --- /dev/null +++ b/packages/react-doctor/tests/score-evidence-opt-out.test.ts @@ -0,0 +1,51 @@ +import * as path from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +const collectScoreEvidenceMock = vi.hoisted(() => vi.fn(() => [])); + +vi.mock("../src/cli/utils/collect-score-evidence.js", () => ({ + collectScoreEvidence: collectScoreEvidenceMock, +})); + +vi.mock("../src/cli/utils/with-run-span.js", () => ({ + recordSentryProjectContext: vi.fn(), + resetSentryRunState: vi.fn(), + withRunSpan: (run: (rootSpan: object) => Promise): Promise => run({}), +})); + +vi.mock("../src/cli/utils/apply-observability.js", () => ({ + applyObservability: (program: Program): Program => program, +})); + +vi.mock("../src/cli/utils/build-run-event.js", () => ({ + recordRunEvent: vi.fn(), +})); + +import { inspect } from "../src/inspect.js"; + +const BASIC_REACT_DIRECTORY = path.resolve( + import.meta.dirname, + "..", + "..", + "core", + "tests", + "fixtures", + "basic-react", +); + +describe("score evidence opt-out", () => { + beforeEach(() => { + collectScoreEvidenceMock.mockClear(); + }); + + it("does not collect evidence when scoring is disabled and telemetry has a root span", async () => { + await inspect(BASIC_REACT_DIRECTORY, { + deadCode: false, + lint: false, + noScore: true, + silent: true, + }); + + expect(collectScoreEvidenceMock).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/react-doctor/tests/telemetry-disclosure.test.ts b/packages/react-doctor/tests/telemetry-disclosure.test.ts new file mode 100644 index 0000000000..ea24826e88 --- /dev/null +++ b/packages/react-doctor/tests/telemetry-disclosure.test.ts @@ -0,0 +1,58 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vite-plus/test"; +import { + TELEMETRY_DISCLOSURE_LINES, + showTelemetryDisclosureIfNeeded, +} from "../src/cli/utils/telemetry-disclosure.js"; + +describe("showTelemetryDisclosureIfNeeded", () => { + let configDirectory: string; + + beforeEach(() => { + configDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "react-doctor-telemetry-notice-")); + }); + + afterEach(() => { + fs.rmSync(configDirectory, { recursive: true, force: true }); + }); + + it("shows the disclosure once for an interactive telemetry-enabled run", () => { + const lines: string[] = []; + const input = { + isInteractive: true, + store: { cwd: configDirectory }, + telemetryEnabled: true, + writeLine: (line: string) => lines.push(line), + }; + + expect(showTelemetryDisclosureIfNeeded(input)).toBe(true); + expect(lines).toEqual(["", ...TELEMETRY_DISCLOSURE_LINES, ""]); + expect(showTelemetryDisclosureIfNeeded(input)).toBe(false); + }); + + it("does not consume the disclosure during a headless run", () => { + const input = { + isInteractive: false, + store: { cwd: configDirectory }, + telemetryEnabled: true, + writeLine: (): void => {}, + }; + + expect(showTelemetryDisclosureIfNeeded(input)).toBe(false); + expect(showTelemetryDisclosureIfNeeded({ ...input, isInteractive: true })).toBe(true); + }); + + it("does not show or consume the disclosure when telemetry is disabled", () => { + const input = { + isInteractive: true, + store: { cwd: configDirectory }, + telemetryEnabled: false, + writeLine: (): void => {}, + }; + + expect(showTelemetryDisclosureIfNeeded(input)).toBe(false); + expect(showTelemetryDisclosureIfNeeded({ ...input, telemetryEnabled: true })).toBe(true); + }); +});