datumbrain · fahadsiddiqui · Jul 21, 2026 · Jul 19, 2026
diff --git a/tests/false-positive-corpus.test.ts b/tests/false-positive-corpus.test.ts
@@ -0,0 +1,66 @@
+import * as path from 'path';
+import { describe, expect, test } from '@jest/globals';
+import { PrivacyScanner } from '../src/scanner/engine';
+import { BUILTIN_RULES, loadExternalRulesFromJson } from '../src/scanner/detectors';
+import { DetectionRule } from '../src/types/findings';
+import {
+  CANARIES,
+  CORPUS,
+  KNOWN_FALSE_POSITIVES,
+} from './fixtures/false-positive-corpus';
+
+// Load the full default rule set exactly as rules.test.ts does: built-in rules
+// plus the external coding-only rules from data/regex_list_1.json.
+const externalRulesPath = path.resolve(process.cwd(), 'data/regex_list_1.json');
+const externalRules = loadExternalRulesFromJson(externalRulesPath, { codingOnly: true });
+const allRules: DetectionRule[] = [...BUILTIN_RULES, ...externalRules];
+const scanner = new PrivacyScanner(allRules);
+
+const corpusCases = CORPUS.flatMap((group) =>
+  group.snippets.map((snippet) => ({ category: group.category, snippet }))
+);
+
+describe('false-positive regression corpus', () => {
+  test('the full default rule set is loaded', () => {
+    // Guard against silently scanning with an empty/partial rule set, which would
+    // make every zero-finding assertion below vacuously pass.
+    expect(BUILTIN_RULES.length).toBeGreaterThan(0);
+    expect(externalRules.length).toBeGreaterThan(0);
+    expect(allRules.length).toBeGreaterThan(BUILTIN_RULES.length);
+  });
+
+  test('the corpus is non-trivial', () => {
+    expect(corpusCases.length).toBeGreaterThanOrEqual(40);
+  });
+
+  test.each(corpusCases)(
+    'produces zero findings [$category]: $snippet',
+    ({ snippet }) => {
+      const result = scanner.scan(snippet);
+      // On failure, surface which rules fired so a regression is easy to diagnose.
+      const fired = result.findings.map((f) => `${f.ruleId}:${JSON.stringify(f.match)}`);
+      expect(fired).toEqual([]);
+    }
+  );
+});
+
+describe('true-positive canaries (scanner self-check)', () => {
+  test.each(CANARIES)('fires $label', ({ snippet, expectRuleId }) => {
+    const result = scanner.scan(snippet);
+    const found = result.findings.some((finding) => finding.ruleId === expectRuleId);
+    expect(found).toBe(true);
+  });
+});
+
+describe('documented current-behavior false positives', () => {
+  // These are NOT bugs this PR fixes; they pin present-day behaviour so that if a
+  // rule is later refined, the change is intentional and visible in this test.
+  test.each(KNOWN_FALSE_POSITIVES)(
+    'still fires $firesRuleId: $snippet',
+    ({ snippet, firesRuleId }) => {
+      const result = scanner.scan(snippet);
+      const found = result.findings.some((finding) => finding.ruleId === firesRuleId);
+      expect(found).toBe(true);
+    }
+  );
+});
diff --git a/tests/fixtures/false-positive-corpus.ts b/tests/fixtures/false-positive-corpus.ts
@@ -0,0 +1,204 @@
+/**
+ * False-positive regression corpus (see issue #20).
+ *
+ * Every string in CORPUS is a realistic, benign snippet that MUST produce zero
+ * findings when scanned with the full default rule set (built-in rules plus the
+ * external coding-only rules from data/regex_list_1.json). The corpus pins the
+ * *current* good behaviour of the scanner: a rule change that starts flagging any
+ * of these snippets is a regression and should fail CI.
+ *
+ * Every snippet here was verified empirically against the compiled scanner before
+ * being added - only strings that genuinely score zero findings today are included.
+ * Near-misses that DO fire under the current rules are documented in
+ * KNOWN_FALSE_POSITIVES below and deliberately kept OUT of the corpus, rather than
+ * changing any rule. This is a test-only pin, not a rule fix.
+ */
+
+export interface CorpusGroup {
+  category: string;
+  snippets: string[];
+}
+
+export const CORPUS: CorpusGroup[] = [
+  {
+    category: 'prose mentioning secret terms (CLI / docs)',
+    snippets: [
+      'To authenticate, pass your API key in the request header.',
+      'Run the CLI and provide a token when prompted for access.',
+      'docs: how to rotate an API key and revoke a token safely',
+      'See the authentication docs for how Bearer tokens and JWTs work.',
+      'Stripe keys start with sk_live_ or sk_test_ prefixes.',
+      'the value looks like ghp_ but is just a prefix mention',
+    ],
+  },
+  {
+    category: 'environment-variable references (no literal secret)',
+    snippets: [
+      'api_key = os.environ["API_KEY"]',
+      'const apiKey = process.env.API_KEY;',
+      'apiKey: process.env.API_KEY',
+      'password = getpass.getpass("Enter your password: ")',
+      'the config uses ${API_KEY} interpolation at runtime',
+      'env: GITHUB_TOKEN is read by the action',
+      'steps: uses actions/checkout@v4 with token: ${{ secrets.GITHUB_TOKEN }}',
+      'heroku_api_key should be set in the dashboard',
+      'facebook_app_secret is configured per environment',
+    ],
+  },
+  {
+    category: 'placeholder / example values',
+    snippets: [
+      'api_key = "YOUR_KEY"',
+      'api_key = "<your-api-key>"',
+      'token: "changeme"',
+      'secret: null',
+      'Authorization: Bearer <token>',
+      'Authorization: Bearer YOUR_TOKEN_HERE',
+      'set the key to sk-xxx or sk-... in your config',
+    ],
+  },
+  {
+    category: 'JWT-lookalikes (not three dot-separated eyJ segments)',
+    snippets: [
+      'the JWT starts with eyJ but this string eyJhbGci is not a full token',
+      'header.payload.signature is the JWT structure',
+      'eyJhbGciOiJIUzI1NiJ9 is only the header segment, not a full JWT',
+      'a.b.c is not a jwt; eyJ.eyJ needs a third segment',
+    ],
+  },
+  {
+    category: 'hashes, UUIDs, git SHAs, and lockfile blobs',
+    snippets: [
+      'The commit 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b was reverted.',
+      'id: 550e8400-e29b-41d4-a716-446655440000',
+      '"integrity": "sha512-abc123DEF456ghi789JKLmno0PQRstu+vwx/yz=="',
+      'the docker image sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+      'commit c8f1e18b78e34a1f9d0c2e5a6b7c8d9e0f1a2b3c',
+      'trace_id=00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01',
+      'resolved "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"',
+      'MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQ is a base64 fragment in a code comment',
+      'data = base64.b64decode("SGVsbG8gV29ybGQgdGhpcyBpcyBhIHRlc3Q=")',
+      'const uuid = crypto.randomUUID();',
+      // A bcrypt hash is not an API-key-shaped secret and is not matched today.
+      "const hash = '$2b$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy';",
+    ],
+  },
+  {
+    category: 'CLI / infra commands and secret-manager references (no actual secret)',
+    snippets: [
+      'kubectl get pods -n production -o wide',
+      'gcloud auth login --brief',
+      'kubectl create secret generic db --from-literal=password=REDACTED --dry-run=client',
+      'gcloud secrets versions access latest --secret=db-password',
+      'reference the secret at projects/my-project/secrets/db-password/versions/latest',
+      'resource "google_secret_manager_secret" "db" { secret_id = "db-password" }',
+      '- old_secret = get("projects/p/secrets/s/versions/1")\n+ new_secret = get("projects/p/secrets/s/versions/2")',
+      'cat /etc/passwd | grep root',
+      'export AWS_PROFILE=default && aws s3 ls',
+      'npm install left-pad@1.3.0 --save-exact',
+      'git log --oneline -n 20 shows recent commits',
+    ],
+  },
+  {
+    category: 'markdown docs about authentication',
+    snippets: [
+      '# Authentication\n\nProvide an API key via the `Authorization` header as a Bearer token.',
+    ],
+  },
+];
+
+/**
+ * KNOWN_FALSE_POSITIVES - realistic benign strings that the CURRENT rules DO flag.
+ *
+ * These are deliberately excluded from CORPUS so it stays green. They document
+ * present-day behaviour discovered while building the corpus; each is a candidate
+ * for a future rule refinement, but this PR only pins behaviour and does not touch
+ * any rule. If a rule is later tightened, move the corresponding entry into CORPUS.
+ */
+export const KNOWN_FALSE_POSITIVES: { snippet: string; firesRuleId: string; note: string }[] = [
+  {
+    snippet: 'AKIAIOSFODNN7EXAMPLE',
+    firesRuleId: 'aws-api-key',
+    note: "Amazon's own documented example access key ID; the aws-api-key AKIA... shape matches it.",
+  },
+  {
+    snippet: 'api_key = "YOUR_API_KEY_HERE"',
+    firesRuleId: 'generic-code-secret-assignment',
+    note: 'The 17-char placeholder is >=16 word chars in quotes after api_key=, so the generic assignment rule fires. "YOUR_KEY" (used in the corpus) is short enough to stay clean.',
+  },
+  {
+    snippet: 'OPENAI_API_KEY=sk-xxxxxxxxxxxxxxxxxxxx',
+    firesRuleId: 'openai-api-key',
+    note: 'sk- followed by 20+ chars matches even when the chars are placeholder x\'s. Shorter "sk-xxx" prose stays clean.',
+  },
+  {
+    snippet: 'the field secret_access_key holds your AWS secret',
+    firesRuleId: 'external-aws-credentials-context',
+    note: 'external-aws-credentials-context is a bare keyword alternation (access_key_id|secret_access_key|...), so the term alone fires with no value present.',
+  },
+  {
+    snippet: 'POST https://login.microsoftonline.com/common/oauth2/v2.0/token',
+    firesRuleId: 'external-microsoft-office-365-oauth-context',
+    note: 'The rule matches the public OAuth endpoint URL itself, which appears in ordinary documentation.',
+  },
+  {
+    snippet: 'root:x:0:0:root:/root:/bin/bash',
+    firesRuleId: 'external-password-etc-passwd',
+    note: 'A standard /etc/passwd line (no password hash - the x is a shadow reference) is matched by the passwd-file rule.',
+  },
+  {
+    snippet: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDexample user@host',
+    firesRuleId: 'external-ssh-rsa-public',
+    note: 'A PUBLIC ssh-rsa key line is matched; public keys are not secret but the rule flags the format.',
+  },
+];
+
+/**
+ * True-positive canaries - one real-shaped secret per built-in category family.
+ * These MUST fire, so the corpus test file can self-check that the full rule set
+ * was actually loaded (a corpus of zero-finding strings would also pass against a
+ * broken/empty scanner). Each entry names the built-in rule expected to match.
+ */
+export const CANARIES: { label: string; snippet: string; expectRuleId: string }[] = [
+  {
+    label: 'email address (pii)',
+    snippet: 'contact user@example.com for access',
+    expectRuleId: 'email-address',
+  },
+  {
+    label: 'JWT (auth-token)',
+    snippet:
+      'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U',
+    expectRuleId: 'jwt-token',
+  },
+  {
+    label: 'Bearer token (auth-token)',
+    snippet: 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9abcdefghij',
+    expectRuleId: 'bearer-token',
+  },
+  {
+    label: 'AWS access key ID (secret)',
+    snippet: 'AKIAIOSFODNN7EXAMPLE',
+    expectRuleId: 'aws-api-key',
+  },
+  {
+    label: 'OpenAI API key (secret)',
+    snippet: 'sk-proj-1234567890abcdefghij',
+    expectRuleId: 'openai-api-key',
+  },
+  {
+    label: 'GitHub token (secret)',
+    snippet: 'ghp_abcdefghijklmnopqrstuvwxyz1234567890',
+    expectRuleId: 'github-token',
+  },
+  {
+    label: 'hardcoded secret assignment (secret)',
+    snippet: 'api_key = "abc1234567890secretvalue"',
+    expectRuleId: 'generic-code-secret-assignment',
+  },
+  {
+    label: 'PEM private key (secret)',
+    snippet: '-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBg...\n-----END PRIVATE KEY-----',
+    expectRuleId: 'private-key',
+  },
+];