feat(tarball): implement new EntryFilesAnalyser API

fraxken · fraxken · commit b516bf966a38 · 2025-05-19T21:33:45.000+02:00
diff --git a/workspaces/scanner/src/index.ts b/workspaces/scanner/src/index.ts
@@ -97,7 +97,7 @@ export async function verify(
       ...NPM_TOKEN, registry: getLocalRegistryURL(), cache: `${os.homedir()}/.npm`
     });
 
-    const scanResult = await tarball.scanPackage(dest, packageName);
+    const scanResult = await tarball.scanPackage(dest);
 
     return scanResult;
   }
diff --git a/workspaces/tarball/src/class/TarballExtractor.class.ts b/workspaces/tarball/src/class/TarballExtractor.class.ts
@@ -0,0 +1,164 @@
+// Import Node.js Dependencies
+import os from "node:os";
+import path from "node:path";
+
+// Import Third-party Dependencies
+import pacote from "pacote";
+import * as conformance from "@nodesecure/conformance";
+import { ManifestManager } from "@nodesecure/mama";
+import {
+  EntryFilesAnalyser,
+  AstAnalyser,
+  type Warning,
+  type Dependency
+} from "@nodesecure/js-x-ray";
+
+// Import Internal Dependencies
+import {
+  getTarballComposition
+} from "../utils/index.js";
+
+// CONSTANTS
+const kNpmToken = typeof process.env.NODE_SECURE_TOKEN === "string" ?
+  { token: process.env.NODE_SECURE_TOKEN } :
+  {};
+
+export interface NpmTarballExtractOptions {
+  registry?: string;
+}
+
+export class TarballExtractor {
+  static JS_EXTENSIONS = new Set([".js", ".mjs", ".cjs"]);
+
+  public manifest: ManifestManager;
+  public archiveLocation: string;
+
+  constructor(
+    archiveLocation: string,
+    mama: ManifestManager
+  ) {
+    this.archiveLocation = archiveLocation;
+    this.manifest = mama;
+  }
+
+  async scan() {
+    const [
+      composition,
+      spdx
+    ] = await Promise.all([
+      getTarballComposition(this.archiveLocation),
+      conformance.extractLicenses(this.archiveLocation)
+    ]);
+
+    return {
+      spdx,
+      composition
+    };
+  }
+
+  async runJavaScriptSast(
+    JSFiles: string[]
+  ) {
+    const astAnalyzer = new AstAnalyser();
+
+    const dependencies: Record<string, Record<string, Dependency>> = Object.create(null);
+    const minified: string[] = [];
+    const warnings: Warning[] = [];
+
+    // TODO: make sure all files exist
+    const entries = [...this.manifest.getEntryFiles()]
+      .filter((entryFile) => TarballExtractor.JS_EXTENSIONS.has(path.extname(entryFile)))
+      .map((relPath) => path.join(this.archiveLocation, relPath));
+
+    if (entries.length > 0) {
+      const efa = new EntryFilesAnalyser({
+        astAnalyzer
+      });
+      for await (const fileReport of efa.analyse(entries)) {
+        warnings.push(
+          ...fileReport.warnings.map((warning) => {
+            return { ...warning, file: fileReport.file };
+          })
+        );
+
+        if (fileReport.ok) {
+          dependencies[fileReport.file] = Object.fromEntries(
+            fileReport.dependencies
+          );
+          if (fileReport.isMinified) {
+            minified.push(fileReport.file);
+          }
+        }
+      }
+    }
+    else {
+      const { name, type = "script" } = this.manifest.document;
+
+      for (const file of JSFiles) {
+        const result = await astAnalyzer.analyseFile(
+          path.join(this.archiveLocation, file),
+          {
+            packageName: name,
+            module: type === "module"
+          }
+        );
+
+        warnings.push(
+          ...result.warnings.map((curr) => Object.assign({}, curr, { file }))
+        );
+        if (result.ok) {
+          dependencies[file] = Object.fromEntries(result.dependencies);
+          if (result.isMinified) {
+            minified.push(file);
+          }
+        }
+      }
+    }
+
+    return {
+      dependencies,
+      warnings,
+      minified
+    };
+  }
+
+  static async fromGit(
+    location: string,
+    url: string,
+    options: NpmTarballExtractOptions = {}
+  ) {
+    const { registry } = options;
+
+    await pacote.extract(url, location, {
+      ...kNpmToken,
+      registry,
+      cache: `${os.homedir()}/.npm`
+    });
+
+    return this.fromFileSystem(location);
+  }
+
+  static async fromNpm(
+    location: string,
+    spec: string,
+    options: NpmTarballExtractOptions = {}
+  ) {
+    const { registry } = options;
+
+    await pacote.extract(spec, location, {
+      ...kNpmToken,
+      registry,
+      cache: `${os.homedir()}/.npm`
+    });
+
+    return this.fromFileSystem(location);
+  }
+
+  static async fromFileSystem(
+    location: string
+  ): Promise<TarballExtractor> {
+    const mama = await ManifestManager.fromPackageJSON(location);
+
+    return new TarballExtractor(location, mama);
+  }
+}
diff --git a/workspaces/tarball/src/tarball.ts b/workspaces/tarball/src/tarball.ts
@@ -1,26 +1,22 @@
 // Import Node.js Dependencies
 import path from "node:path";
-import os from "node:os";
 
 // Import Third-party Dependencies
 import {
-  AstAnalyser,
   type Warning,
   type Dependency
 } from "@nodesecure/js-x-ray";
-import pacote from "pacote";
 import * as conformance from "@nodesecure/conformance";
-import { ManifestManager } from "@nodesecure/mama";
 
 // Import Internal Dependencies
 import {
-  getTarballComposition,
   isSensitiveFile,
   analyzeDependencies,
+  filterDependencyKind,
   booleanToFlags
 } from "./utils/index.js";
+import { TarballExtractor } from "./class/TarballExtractor.class.js";
 import * as warnings from "./warnings.js";
-import * as sast from "./sast/index.js";
 
 export interface DependencyRef {
   id: number;
@@ -53,12 +49,7 @@ export interface DependencyRef {
 }
 
 // CONSTANTS
-const NPM_TOKEN = typeof process.env.NODE_SECURE_TOKEN === "string" ?
-  { token: process.env.NODE_SECURE_TOKEN } :
-  {};
-
 const kNativeCodeExtensions = new Set([".gyp", ".c", ".cpp", ".node", ".so", ".h"]);
-const kJsExtname = new Set([".js", ".mjs", ".cjs"]);
 
 export interface scanDirOrArchiveOptions {
   ref: DependencyRef;
@@ -73,33 +64,22 @@ export async function scanDirOrArchive(
   options: scanDirOrArchiveOptions
 ) {
   const { ref, location = process.cwd(), tmpLocation = null, registry } = options;
+  const spec = `${name}@${version}`;
 
-  const isNpmTarball = !(tmpLocation === null);
-  const dest = isNpmTarball ? path.join(tmpLocation, `${name}@${version}`) : location;
-
-  // If this is an NPM tarball then we extract it on the disk with pacote.
-  if (isNpmTarball) {
-    await pacote.extract(
-      ref.flags.includes("isGit") ? ref.gitUrl! : `${name}@${version}`,
-      dest,
-      {
-        ...NPM_TOKEN,
-        registry,
-        cache: `${os.homedir()}/.npm`
-      }
-    );
+  let tarex: TarballExtractor;
+  if (typeof tmpLocation === "string") {
+    const location = path.join(tmpLocation, spec);
+
+    tarex = ref.flags.includes("isGit") ?
+      await TarballExtractor.fromGit(location, ref.gitUrl!, { registry }) :
+      await TarballExtractor.fromNpm(location, spec, { registry });
   }
+  else {
+    tarex = await TarballExtractor.fromFileSystem(location);
+  }
+  const mama = tarex.manifest;
 
-  // Read the package.json at the root of the directory or archive.
-  const [
-    mama,
-    composition,
-    spdx
-  ] = await Promise.all([
-    ManifestManager.fromPackageJSON(dest),
-    getTarballComposition(dest),
-    conformance.extractLicenses(dest)
-  ]);
+  const { composition, spdx } = await tarex.scan();
 
   {
     const { description, engines, repository, scripts } = mama.document;
@@ -119,17 +99,34 @@ export async function scanDirOrArchive(
 
   // Search for minified and runtime dependencies
   // Run a JS-X-Ray analysis on each JavaScript files of the project!
-  const scannedFiles = await sast.scanManyFiles(composition.files, dest, name);
+  const scannedFiles = await tarex.runJavaScriptSast(
+    composition.files.filter(
+      (name) => TarballExtractor.JS_EXTENSIONS.has(path.extname(name))
+    )
+  );
 
-  ref.warnings.push(...scannedFiles.flatMap((row) => row.warnings));
-  if (/^0(\.\d+)*$/.test(version)) {
+  ref.warnings.push(...scannedFiles.warnings);
+  if (mama.hasZeroSemver) {
     ref.warnings.push(warnings.getSemVerWarning(version));
   }
 
-  const dependencies = [...new Set(scannedFiles.flatMap((row) => row.dependencies))];
-  const filesDependencies = [...new Set(scannedFiles.flatMap((row) => row.filesDependencies))];
-  const tryDependencies = new Set(scannedFiles.flatMap((row) => row.tryDependencies));
-  const minifiedFiles = scannedFiles.filter((row) => row.isMinified).flatMap((row) => row.file);
+  const files = new Set<string>();
+  const dependencies = new Set<string>();
+  const dependenciesInTryBlock = new Set<string>();
+
+  for (const [file, fileDeps] of Object.entries(scannedFiles.dependencies)) {
+    const filtered = filterDependencyKind(
+      [...Object.keys(fileDeps)],
+      path.dirname(file)
+    );
+
+    [...Object.entries(fileDeps)]
+      .flatMap(([name, dependency]) => (dependency.inTry ? [name] : []))
+      .forEach((name) => dependenciesInTryBlock.add(name));
+
+    filtered.packages.forEach((name) => dependencies.add(name));
+    filtered.files.forEach((file) => files.add(file));
+  }
 
   const {
     nodeDependencies,
@@ -139,8 +136,8 @@ export async function scanDirOrArchive(
     unusedDependencies,
     flags
   } = analyzeDependencies(
-    dependencies,
-    { mama, tryDependencies }
+    [...dependencies],
+    { mama, tryDependencies: dependenciesInTryBlock }
   );
 
   ref.size = composition.size;
@@ -150,15 +147,15 @@ export async function scanDirOrArchive(
   ref.composition.required_subpath = subpathImportsDependencies;
   ref.composition.unused.push(...unusedDependencies);
   ref.composition.missing.push(...missingDependencies);
-  ref.composition.required_files = filesDependencies;
+  ref.composition.required_files = [...files];
   ref.composition.required_nodejs = nodeDependencies;
-  ref.composition.minified = minifiedFiles;
+  ref.composition.minified = scannedFiles.minified;
 
   ref.flags.push(...booleanToFlags({
     ...flags,
     hasNoLicense: spdx.uniqueLicenseIds.length === 0,
     hasMultipleLicenses: spdx.uniqueLicenseIds.length > 1,
-    hasMinifiedCode: minifiedFiles.length > 0,
+    hasMinifiedCode: scannedFiles.minified.length > 0,
     hasWarnings: ref.warnings.length > 0 && !ref.flags.includes("hasWarnings"),
     hasBannedFile: composition.files.some((path) => isSensitiveFile(path)),
     hasNativeCode: mama.flags.isNative ||
@@ -189,46 +186,24 @@ export interface ScannedPackageResult {
 }
 
 export async function scanPackage(
-  dest: string,
-  packageName?: string
+  dest: string
 ): Promise<ScannedPackageResult> {
-  const [
-    mama,
+  const extractor = await TarballExtractor.fromFileSystem(dest);
+
+  const {
     composition,
     spdx
-  ] = await Promise.all([
-    ManifestManager.fromPackageJSON(dest),
-    getTarballComposition(dest),
-    conformance.extractLicenses(dest)
-  ]);
-  const { type = "script" } = mama.document;
-
-  // Search for runtime dependencies
-  const dependencies: Record<string, Record<string, Dependency>> = Object.create(null);
-  const minified: string[] = [];
-  const warnings: Warning[] = [];
-
-  const JSFiles = composition.files
-    .filter((name) => kJsExtname.has(path.extname(name)));
-  for (const file of JSFiles) {
-    const result = await new AstAnalyser().analyseFile(
-      path.join(dest, file),
-      {
-        packageName: packageName ?? mama.document.name,
-        module: type === "module"
-      }
-    );
+  } = await extractor.scan();
 
-    warnings.push(
-      ...result.warnings.map((curr) => Object.assign({}, curr, { file }))
-    );
-    if (result.ok) {
-      dependencies[file] = Object.fromEntries(result.dependencies);
-      if (result.isMinified) {
-        minified.push(file);
-      }
-    }
-  }
+  const {
+    dependencies,
+    warnings,
+    minified
+  } = await extractor.runJavaScriptSast(
+    composition.files.filter(
+      (name) => TarballExtractor.JS_EXTENSIONS.has(path.extname(name))
+    )
+  );
 
   return {
     files: {

Original file line number	Diff line number	Diff line change
`@@ -97,7 +97,7 @@ export async function verify(`
`97`	`97`	...NPM_TOKEN, registry: getLocalRegistryURL(), cache: `${os.homedir()}/.npm`
`98`	`98`	`});`
`99`	`99`
`100`		`- const scanResult = await tarball.scanPackage(dest, packageName);`
	`100`	`+ const scanResult = await tarball.scanPackage(dest);`
`101`	`101`
`102`	`102`	`return scanResult;`
`103`	`103`	`}`