refactor(scan): extract the scan engine into @threatcrush/scan

apps/cli/package.json

@@ -36,6 +36,7 @@
   "dependencies": {
     "@iarna/toml": "^2.2.5",
     "@sentry/node": "^8.45.0",
+    "@threatcrush/scan": "workspace:*",
     "better-sqlite3": "^11.7.0",
     "blessed": "^0.1.81",
     "blessed-contrib": "^4.11.0",

apps/cli/src/commands/__tests__/scan.test.ts

@@ -0,0 +1,21 @@
+import { describe, expect, it } from 'vitest';
+import { parseFailOn } from '../scan.js';
+
+/**
+ * Flag parsing lives here rather than with the engine tests.
+ *
+ * `@threatcrush/scan` decides what a finding is and whether a set of them
+ * clears a threshold; it has no opinion about argv. This is the boundary the
+ * package extraction drew, and the test placement follows it.
+ */
+describe('--fail-on', () => {
+  it('accepts a comma-separated list of severities', () => {
+    expect(parseFailOn('critical,high')).toEqual(['critical', 'high']);
+  });
+
+  it('rejects an unknown severity rather than silently ignoring it', () => {
+    // Silently accepting `--fail-on hihg` produces a gate that never fires,
+    // which looks exactly like a passing build.
+    expect(() => parseFailOn('hihg')).toThrow(/unknown severity/);
+  });
+});

apps/cli/src/commands/scan.ts

@@ -5,11 +5,9 @@ import ora from 'ora';
 import { banner, logger } from '../core/logger.js';
 import type { RunResult, StructuredFinding } from '../core/run-result.js';
 import { summarize } from '../core/run-result.js';
-import { scanDependencies } from '../scan/dependencies.js';
-import { meetsFailThreshold, scanPath } from '../scan/engine.js';
-import { buildSarif } from '../scan/sarif.js';
-import type { ScanFinding, Severity } from '../scan/types.js';
-import { SEVERITY_ORDER } from '../scan/types.js';
+import { meetsFailThreshold, SEVERITY_ORDER } from '@threatcrush/scan';
+import type { ScanFinding, Severity } from '@threatcrush/scan';
+import { buildSarif, scanDependencies, scanPath } from '@threatcrush/scan/node';
 
 export type ScanFormat = 'text' | 'json' | 'sarif';
 

apps/cli/tsup.config.ts

@@ -17,7 +17,11 @@ export default defineConfig({
   dts: false,
 
   external: ['better-sqlite3', 'blessed', 'blessed-contrib', 'react', 'react-blessed', 'react-blessed-contrib'],
-  noExternal: ['chalk', 'ora', '@iarna/toml', 'commander'],
+  // `@threatcrush/scan` is bundled, not externalised. It resolves to
+  // TypeScript source rather than a build output — see its package.json — so
+  // there is nothing for Node to require at runtime, and the published CLI
+  // must not gain a dependency on a package that is not published.
+  noExternal: ['chalk', 'ora', '@iarna/toml', 'commander', '@threatcrush/scan'],
 
   async onSuccess() {
     // Ship the systemd unit template alongside the compiled bundle.

packages/scan/README.md

@@ -0,0 +1,65 @@
+# `@threatcrush/scan`
+
+The scan rules and engine, shared by the CLI, web, desktop and extension.
+
+Previously this lived at `apps/cli/src/scan/`, which meant the CLI was the only
+surface that could run a scan. Every other app either did without or would have
+grown its own copy of the rules.
+
+## Two entry points
+
+```ts
+import { scanText, CODE_RULES } from '@threatcrush/scan';        // anywhere
+import { scanPath, buildSarif } from '@threatcrush/scan/node';   // needs a filesystem
+```
+
+| entry | contains | runs in |
+|---|---|---|
+| `.` | rules, `scanText`, language detection, suppressions, severity | browser, worker, Node |
+| `./node` | `scanPath` tree walker, dependency scan, SARIF output | Node only |
+
+The default entry point imports nothing from `node:`. That is enforced by
+`src/__tests__/boundaries.test.ts`, not by convention — a browser bundle breaks
+at the *consumer's* build if a filesystem import creeps in, which is a failure
+that surfaces late and in the wrong repository.
+
+Verify by hand at any time:
+
+```sh
+npx esbuild src/index.ts --bundle --platform=browser --format=esm --outfile=/dev/null   # succeeds
+npx esbuild src/node/index.ts --bundle --platform=browser --format=esm --outfile=/dev/null  # fails, by design
+```
+
+## Why `exports` points at TypeScript source
+
+This is an internal package: `exports` resolves to `src/*.ts` rather than a
+build output.
+
+The alternative — compiling to `dist/` — introduces a build ordering
+requirement, and the release workflow runs `pnpm --filter @profullstack/threatcrush build`
+alone. A package that had to be built first would publish a broken CLI the
+first time someone forgot, and the failure would be a runtime `MODULE_NOT_FOUND`
+in the published artefact rather than a red build.
+
+Consumers therefore transpile it themselves:
+
+- **CLI** — bundled by tsup via `noExternal`, so the published package stays
+  self-contained and gains no dependency on an unpublished package.
+- **Next.js** (web) — add `transpilePackages: ['@threatcrush/scan']`.
+- **Vite** (desktop, extension) — works as-is; Vite transpiles linked workspace
+  sources by default.
+
+If this package is ever published standalone, add a build step and switch
+`exports` to `dist` with a `publishConfig` override. Nothing else needs to move.
+
+## Adding a rule
+
+Rules live in `src/code-rules.ts`, credentials in `src/secret-rules.ts`,
+manifests in `src/manifest-rules.ts`. Two invariants are enforced by tests:
+
+- **Every language the scanner claims must have at least one rule.** `shell`
+  and `php` were both listed as supported while no rule targeted them, so those
+  files were read and reported clean whatever they contained.
+- **Every rule is tested against the corrected shape as well as the vulnerable
+  one.** A rule that only fires on bad code is untested against the good code
+  standing next to it, which is where false positives come from.

packages/scan/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "@threatcrush/scan",
+  "version": "0.7.0",
+  "description": "ThreatCrush scan rules and engine, shared by the CLI, web, desktop and extension.",
+  "license": "MIT",
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts",
+    "./node": "./src/node/index.ts"
+  },
+  "files": [
+    "src"
+  ],
+  "scripts": {
+    "test": "vitest run",
+    "typecheck": "tsc --noEmit"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.1",
+    "typescript": "^5.6.3",
+    "vitest": "^3.2.4"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/profullstack/threatcrush.git",
+    "directory": "packages/scan"
+  },
+  "homepage": "https://threatcrush.com"
+}

packages/scan/src/__tests__/boundaries.test.ts

@@ -0,0 +1,58 @@
+import { readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { describe, expect, it } from 'vitest';
+
+/**
+ * The default entry point must stay free of `node:` imports.
+ *
+ * This is the whole reason the package is split the way it is. The web app,
+ * the extension and the desktop renderer import `@threatcrush/scan` into a
+ * browser bundle; a single `node:fs` anywhere in that module graph breaks all
+ * of them, and it breaks at *their* build, not ours — which is the kind of
+ * failure that gets found late and blamed on the wrong repository.
+ *
+ * A comment saying "do not import node: here" does not survive contact with a
+ * hurried change. This does.
+ */
+
+const SRC = join(__dirname, '..');
+
+function sourceFilesOutsideNodeEntry(dir: string, acc: string[] = []): string[] {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    // `node/` is the entry point that is *allowed* to touch the filesystem,
+    // and `__tests__/` is this file and its neighbours — neither ships to a
+    // browser.
+    if (entry.isDirectory()) {
+      if (entry.name === 'node' || entry.name === '__tests__') continue;
+      sourceFilesOutsideNodeEntry(join(dir, entry.name), acc);
+      continue;
+    }
+    if (entry.name.endsWith('.ts')) acc.push(join(dir, entry.name));
+  }
+  return acc;
+}
+
+describe('module boundaries', () => {
+  const files = sourceFilesOutsideNodeEntry(SRC);
+
+  it('finds the source files it is supposed to be checking', () => {
+    // Guards the guard: a broken walk would make every assertion below pass
+    // over an empty list.
+    expect(files.length).toBeGreaterThanOrEqual(5);
+  });
+
+  it.each(files.map((f) => [f.slice(SRC.length + 1), f] as const))(
+    '%s imports nothing from node:',
+    (_label, file) => {
+      const offending = readFileSync(file, 'utf-8')
+        .split('\n')
+        // Comment lines are skipped, because the first thing this test found
+        // was the sentence in `text.ts` explaining why `node:fs` must not
+        // appear there. Matching prose as if it were code is the exact bug the
+        // scanner's own `proseLines` exists to avoid.
+        .filter((line) => !/^\s*(?:\/\/|\/?\*)/.test(line))
+        .filter((line) => /\bfrom\s+['"]node:/.test(line) || /\brequire\(\s*['"]node:/.test(line));
+      expect(offending).toEqual([]);
+    },
+  );
+});

apps/cli/src/scan/__tests__/code-rules.test.ts → packages/scan/src/__tests__/code-rules.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from 'vitest';
 import { CODE_RULES, proseLines } from '../code-rules.js';
-import { languageOf, scanText } from '../engine.js';
+import { languageOf, scanText } from '../text.js';
 import type { ScanLanguage } from '../types.js';
 
 /**

apps/cli/src/scan/__tests__/sarif.test.ts → packages/scan/src/__tests__/sarif.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { buildSarif, fingerprintOf, sarifLevel, securitySeverity, toArtifactUri } from '../sarif.js';
+import { buildSarif, fingerprintOf, sarifLevel, securitySeverity, toArtifactUri } from '../node/sarif.js';
 import type { ScanFinding } from '../types.js';
 
 const finding = (overrides: Partial<ScanFinding> = {}): ScanFinding => ({

apps/cli/src/scan/__tests__/engine.test.ts → packages/scan/src/__tests__/text.test.ts

@@ -1,12 +1,11 @@
 import { describe, expect, it } from 'vitest';
-import { parseFailOn } from '../../commands/scan.js';
 import {
   collectSuppressions,
   languageOf,
   languageOfShebang,
   meetsFailThreshold,
   scanText,
-} from '../engine.js';
+} from '../text.js';
 import { detectTyposquat, editDistance, scanPackageJson, scanRequirementsTxt } from '../manifest-rules.js';
 import { isKnownPlaceholder, redactSecret } from '../secret-rules.js';
 import type { ScanFinding } from '../types.js';
@@ -130,12 +129,9 @@ describe('--fail-on', () => {
     expect(meetsFailThreshold([at('critical')], [])).toBe(false);
   });
 
-  it('rejects an unknown severity rather than silently ignoring it', () => {
-    // Silently accepting `--fail-on hihg` produces a gate that never fires,
-    // which looks exactly like a passing build.
-    expect(parseFailOn('critical,high')).toEqual(['critical', 'high']);
-    expect(() => parseFailOn('hihg')).toThrow(/unknown severity/);
-  });
+  // Parsing the flag itself is the CLI's job and is tested there — see
+  // apps/cli/src/commands/__tests__/scan.test.ts. This package has no opinion
+  // about argv.
 });
 
 describe('typosquat detection', () => {

packages/scan/src/index.ts

@@ -0,0 +1,37 @@
+/**
+ * `@threatcrush/scan` — the rules and the text engine.
+ *
+ * This entry point is deliberately free of `node:` imports. Everything here
+ * runs in a browser, a service worker or a Node process alike, because the
+ * whole of it operates on strings that somebody else obtained. That is what
+ * lets the web app, the extension and the desktop renderer share one copy of
+ * the rules with the CLI rather than growing their own.
+ *
+ * Anything needing a filesystem — walking a tree, reading a manifest off disk,
+ * writing SARIF — lives behind `@threatcrush/scan/node`.
+ */
+
+export { CODE_RULES, evaluateRule, GENERIC_GUARD, proseLines, untrustedPatternFor } from './code-rules.js';
+export type { CodeRule } from './code-rules.js';
+
+export { scanPackageJson, scanRequirementsTxt, detectTyposquat, editDistance } from './manifest-rules.js';
+export type { ManifestFinding, SquatVerdict } from './manifest-rules.js';
+
+export { isKnownPlaceholder, redactSecret, SECRET_RULES, SENSITIVE_FILES } from './secret-rules.js';
+
+export {
+  collectSuppressions,
+  isTestPath,
+  languageOf,
+  languageOfShebang,
+  meetsFailThreshold,
+  peakSeverity,
+  SCAN_EXTENSIONS,
+  scanManifest,
+  scanText,
+  SKIP_DIRS,
+} from './text.js';
+export type { Suppressions } from './text.js';
+
+export { severityRank, SEVERITY_ORDER } from './types.js';
+export type { Confidence, ScanFinding, ScanLanguage, Severity } from './types.js';

apps/cli/src/scan/dependencies.ts → packages/scan/src/node/dependencies.ts

@@ -10,7 +10,7 @@
 
 import { existsSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
-import type { ScanFinding, Severity } from './types.js';
+import type { ScanFinding, Severity } from '../types.js';
 
 interface OsvVulnerability {
   id: string;

packages/scan/src/node/index.ts

@@ -0,0 +1,15 @@
+/**
+ * `@threatcrush/scan/node` — the parts that need a filesystem.
+ *
+ * Kept apart from the default entry point so that importing the rules does not
+ * drag `node:fs` into a browser bundle. Import from here only where a real
+ * filesystem exists: the CLI, the daemon, a server route.
+ */
+
+export { scanPath } from './walk.js';
+export type { ScanOptions, ScanReport } from './walk.js';
+
+export { scanDependencies } from './dependencies.js';
+
+export { buildSarif, fingerprintOf, sarifLevel, securitySeverity, toArtifactUri } from './sarif.js';
+export type { SarifOptions } from './sarif.js';

apps/cli/src/scan/sarif.ts → packages/scan/src/node/sarif.ts

@@ -26,7 +26,7 @@
 
 import { createHash } from 'node:crypto';
 import { isAbsolute, relative, resolve, sep } from 'node:path';
-import type { ScanFinding, Severity } from './types.js';
+import type { ScanFinding, Severity } from '../types.js';
 
 /**
  * The key our fingerprint is published under.