refactor(modules): fold dep-scanner into code-scanner as the deps subsystem

modules/dep-scanner/README.md → modules/code-scanner/README.md

@@ -1,15 +1,31 @@
-# dep-scanner
+# code-scanner
 
-Detects vulnerable and malicious dependencies on **running servers**.
+Static analysis on codebases. `PRD.md` scopes this module to "vulnerabilities,
+secrets, misconfigs, dependency CVEs" — four different detection models over the
+same directories, so the module is a thin host and each one is a **subsystem**:
 
-Implements [PRD 0002](../../prd/0002-detect-vulnerable-and-malicious-dependencies-on-running-servers.md).
+| Subsystem | Covers | Status |
+| --- | --- | --- |
+| `deps/` | Dependency advisories, malicious install scripts, lockfile drift | **implemented** ([PRD 0002](../../prd/0002-detect-vulnerable-and-malicious-dependencies-on-running-servers.md)) |
+| `secrets/` | Hardcoded credentials | not yet built |
+| `sast/` | Source-level vulnerability analysis | not yet built |
+| `config/` | Misconfiguration checks | not yet built |
 
 ```bash
-threatcrush modules install dep-scanner
+threatcrush modules install code-scanner
 threatcrush scan --deps /srv/app
 ```
 
-## Why another dependency scanner
+PRD 0002 originally proposed the dependency work as a standalone `dep-scanner`
+module and left the boundary open. It is resolved in favour of folding in: two
+modules would have walked the same trees, held two inventories of the same
+packages, and given the operator two path lists to keep in sync — for a
+user-visible surface (`threatcrush scan --deps`) that was always meant to be one
+command. What survives the fold is the *internal* separation, because `deps/`
+owns an external advisory database and a per-ecosystem parser surface that has
+nothing in common with a secrets regex pass.
+
+## Why the deps subsystem exists
 
 Two failures motivated this module, and both are things the existing tools do
 badly or cannot do at all.
@@ -27,7 +43,7 @@ Every number was wrong. It read the root `package.json`, never resolved
 `pnpm-workspace.yaml`, found nothing, and called the nothing *good*. For a
 productivity tool that wastes an afternoon; for a security tool it is a clean
 bill of health issued without an examination. Run against the same workspace,
-this module reports **21 manifests, 59 packages**, and marks the root `partial`
+this subsystem reports **21 project roots and 101 packages**, and marks the root `partial`
 with the reason — because a pnpm lockfile is not parsed yet, and saying so is
 the entire point.
 

modules/dep-scanner/config/example.conf.toml → modules/code-scanner/config/example.conf.toml

@@ -1,4 +1,4 @@
-# Example: /etc/threatcrush/threatcrushd.conf.d/dep-scanner.conf
+# Example: /etc/threatcrush/threatcrushd.conf.d/code-scanner.conf
 #
 # Values here override [module.config.defaults] in mod.toml.
 
@@ -27,7 +27,7 @@ fail_on_unparseable = true
 # Set false on an air-gapped host: install-script scoring and lockfile drift
 # detection both keep working without network access, and they are the two
 # detectors that catch a malicious package before an advisory exists.
-advisories_enabled = true
+deps_advisories = true
 
 # Alerting floor. Findings below this are still counted and logged, just not
 # alerted on.
@@ -40,7 +40,7 @@ min_severity = "medium"
 # There is deliberately no "ignore" — a finding nobody ever sees is
 # indistinguishable from one that was missed. Build tooling still executes on
 # your build host, and that host usually holds deploy credentials.
-dev_dependencies = "rank_down"
+deps_dev_dependencies = "rank_down"
 
 # --- Behavioural detection -------------------------------------------------
 # Score preinstall/install/postinstall scripts on what they do: fetching remote
@@ -49,7 +49,7 @@ dev_dependencies = "rank_down"
 # This is the detector that does not wait for a CVE. event-stream (2018),
 # ua-parser-js (2021) and node-ipc (2022) were all advisory-clean at the moment
 # they were installed; all three executed from a lifecycle script.
-install_scripts = true
+deps_install_scripts = true
 
 # --- Reporting -------------------------------------------------------------
 # Cap alerts per scan so the first run on a long-neglected host summarizes

modules/code-scanner/mod.toml

@@ -0,0 +1,63 @@
+[module]
+name = "code-scanner"
+version = "0.1.0"
+description = "Static analysis on codebases — dependency advisories, malicious install scripts, and lockfile drift"
+author = "Profullstack, Inc."
+license = "MIT"
+homepage = "https://github.com/profullstack/threatcrush/tree/master/modules/code-scanner"
+
+[module.pricing]
+type = "free"
+
+[module.requirements]
+threatcrush = ">=0.2.0"
+os = ["linux", "darwin"]
+capabilities = ["fs:read", "network:outbound"]
+
+[module.config.defaults]
+enabled = true
+
+# Where to look for code. Each path is walked for project roots, skipping
+# node_modules, VCS and build directories. Shared by every subsystem.
+paths = ["/srv", "/var/www", "/opt"]
+
+# How deep to walk below each configured path when looking for project roots.
+max_depth = 4
+
+# Dependencies change on deploy, not by the minute. Six hours keeps advisory
+# data fresh without turning the scanner into background load.
+scan_interval_seconds = 21600
+
+# Alerting floor, shared by every subsystem. Findings below this are still
+# counted and logged, just not alerted on.
+min_severity = "medium"
+
+# Cap alerts per scan so a first run on a neglected host summarizes instead of
+# flooding the alert channel. Findings over the cap carry to the next scan.
+max_alerts = 25
+
+# --- Honesty ---------------------------------------------------------------
+# Alert when a project could not be parsed at all. Leave this on: the failure
+# this scanner was written to prevent is reporting "nothing found" when the
+# truth is "nothing was read". An unexamined project is not a clean one.
+fail_on_unparseable = true
+
+# --- deps subsystem: dependency and supply-chain scanning (PRD 0002) -------
+deps_enabled = true
+
+# Look up installed versions against OSV.dev (aggregates GHSA, PyPA, RustSec,
+# Go vulndb). Set false for a fully offline host; install-script scoring and
+# drift detection continue to work without network access.
+deps_advisories = true
+
+# Score preinstall/install/postinstall scripts on what they do. This is what
+# catches a malicious package before an advisory exists, which is the window in
+# which event-stream, ua-parser-js and node-ipc all did their damage.
+deps_install_scripts = true
+
+# How to treat findings reachable only through devDependencies:
+#   rank_down  demote one severity level (default)
+#   equal      treat identically to runtime dependencies
+# There is deliberately no "ignore": a finding an operator never sees is
+# indistinguishable from one that was missed.
+deps_dev_dependencies = "rank_down"

modules/dep-scanner/package.json → modules/code-scanner/package.json

@@ -1,17 +1,17 @@
 {
-  "name": "threatcrush-module-dep-scanner",
+  "name": "threatcrush-module-code-scanner",
   "version": "0.1.0",
-  "description": "Scans installed dependencies for known advisories, malicious install scripts and lockfile drift",
+  "description": "Static analysis on codebases — dependency advisories, malicious install scripts, and lockfile drift",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "license": "MIT",
   "author": "Profullstack, Inc.",
-  "homepage": "https://threatcrush.com/store/dep-scanner",
+  "homepage": "https://threatcrush.com/store/code-scanner",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/profullstack/threatcrush.git",
-    "directory": "modules/dep-scanner"
+    "directory": "modules/code-scanner"
   },
   "keywords": [
     "threatcrush",

modules/dep-scanner/src/__tests__/inventory.test.ts → modules/code-scanner/src/__tests__/inventory.test.ts

@@ -8,12 +8,12 @@ import {
   inventoryRoot,
   parsePackageLock,
   readWorkspacePatterns,
-} from '../inventory.js';
+} from '../deps/inventory.js';
 
 let dir: string;
 
 beforeEach(async () => {
-  dir = await mkdtemp(join(tmpdir(), 'dep-scanner-'));
+  dir = await mkdtemp(join(tmpdir(), 'code-scanner-'));
 });
 
 afterEach(async () => {

modules/dep-scanner/src/__tests__/scripts.test.ts → modules/code-scanner/src/__tests__/scripts.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { extractScripts, rankScripts, scoreScript, type InstallScript } from '../scripts.js';
+import { extractScripts, rankScripts, scoreScript, type InstallScript } from '../deps/scripts.js';
 
 const script = (command: string, stage: InstallScript['stage'] = 'postinstall'): InstallScript => ({
   packageName: 'example',

modules/dep-scanner/src/__tests__/semver.test.ts → modules/code-scanner/src/__tests__/semver.test.ts

@@ -4,7 +4,7 @@ import {
   fixedVersionFor,
   isVersionAffected,
   parseVersion,
-} from '../semver.js';
+} from '../deps/semver.js';
 
 /**
  * Version comparison is the load-bearing primitive: every advisory match runs

modules/code-scanner/src/deps/index.ts

@@ -0,0 +1,131 @@
+/**
+ * Dependency and supply-chain scanning — the `deps` subsystem of
+ * `code-scanner`.
+ *
+ * Implements PRD 0002, which originally proposed this as a standalone
+ * `dep-scanner` module and left the boundary open. It is folded in here
+ * because `PRD.md` already assigns "dependency CVEs" to `code-scanner`, and a
+ * separate module would have meant two things scanning the same directories,
+ * holding two inventories of the same tree, and two places for an operator to
+ * configure a path list.
+ *
+ * The subsystem keeps its own directory rather than dissolving into the
+ * module: it owns an external advisory database, a per-ecosystem parser
+ * surface that will keep growing, and a detection model (a package graph)
+ * that has nothing in common with the source-level scanning `code-scanner`
+ * will add for secrets and SAST. Those are separable concerns that happen to
+ * share a filesystem walk.
+ */
+
+export * from './inventory.js';
+export * from './osv.js';
+export * from './scripts.js';
+export * from './semver.js';
+
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+import { buildInventory, type Inventory } from './inventory.js';
+import { queryAdvisories, severityRank, type Finding } from './osv.js';
+import { extractScripts, rankScripts, type ScriptVerdict } from './scripts.js';
+
+export interface DepsOptions {
+  paths: readonly string[];
+  maxDepth: number;
+  advisories: boolean;
+  installScripts: boolean;
+  /** `rank_down` demotes dev-only findings; `equal` leaves them alone. */
+  devDependencies: 'rank_down' | 'equal';
+  /** Called when advisory lookup fails, so the caller can surface it. */
+  onDegraded?: (reason: string) => void;
+}
+
+export interface DepsResult {
+  inventory: Inventory;
+  findings: Finding[];
+  scripts: ScriptVerdict[];
+  drift: { root: string; name: string; locked: string; installed: string }[];
+  /** True when any root failed to parse — the caller must not report "clean". */
+  incomplete: boolean;
+}
+
+export const DEPS_DEFAULTS: DepsOptions = {
+  paths: ['/srv', '/var/www', '/opt'],
+  maxDepth: 4,
+  advisories: true,
+  installScripts: true,
+  devDependencies: 'rank_down',
+};
+
+/** One step down the severity ladder, floored at low. */
+export function demote(severity: Finding['severity']): Finding['severity'] {
+  const order: Finding['severity'][] = ['info', 'low', 'medium', 'high', 'critical'];
+  return order[Math.max(1, order.indexOf(severity) - 1)]!;
+}
+
+/**
+ * Run the whole dependency pass: inventory, advisories, install scripts, drift.
+ *
+ * Pure with respect to the daemon — it takes options and returns a result,
+ * with no `ModuleContext` — so the same entry point serves the scheduled
+ * daemon scan and the one-shot `threatcrush scan --deps`.
+ */
+export async function scanDependencies(options: DepsOptions): Promise<DepsResult> {
+  const inventory = await buildInventory(options.paths, options.maxDepth);
+  const scripts = options.installScripts ? await collectInstallScripts(inventory) : [];
+
+  let findings: Finding[] = [];
+  if (options.advisories && inventory.packages.length > 0) {
+    try {
+      findings = await queryAdvisories(
+        inventory.packages.map((p) => ({ name: p.name, version: p.version })),
+      );
+    } catch (err) {
+      // Advisory lookup is the only part needing the network. Losing it must
+      // degrade the scan, not void it — and must be visible, because a silent
+      // skip is indistinguishable from "nothing found".
+      options.onDegraded?.(String(err));
+      inventory.incomplete = true;
+    }
+  }
+
+  // Dev-only findings are ranked down, never dropped (PRD 0002 R9).
+  if (options.devDependencies !== 'equal') {
+    const devOnly = new Set(
+      inventory.packages.filter((p) => p.dev).map((p) => `${p.name}@${p.version}`),
+    );
+    findings = findings.map((f) =>
+      devOnly.has(`${f.package}@${f.version}`) && severityRank(f.severity) > 1
+        ? { ...f, severity: demote(f.severity) }
+        : f,
+    );
+  }
+
+  const drift = inventory.roots.flatMap((root) =>
+    root.drift.map((entry) => ({ root: root.root, ...entry })),
+  );
+
+  return { inventory, findings, scripts, drift, incomplete: inventory.incomplete };
+}
+
+/** Read lifecycle scripts out of every installed package and score them. */
+async function collectInstallScripts(inventory: Inventory): Promise<ScriptVerdict[]> {
+  const scripts = [];
+
+  for (const report of inventory.roots) {
+    for (const record of report.packages) {
+      if (!record.path.startsWith('node_modules')) continue;
+      try {
+        const manifest = JSON.parse(
+          await readFile(join(report.root, record.path, 'package.json'), 'utf8'),
+        ) as { name?: string; version?: string; scripts?: Record<string, string> };
+        scripts.push(...extractScripts(manifest, record.name));
+      } catch {
+        // A package whose manifest cannot be read contributes no scripts; the
+        // root's own parse status already records the trouble.
+      }
+    }
+  }
+
+  return rankScripts(scripts);
+}