fix: nest files with timestamp for plugin runners

packages/core/src/lib/collect-and-persist.ts

@@ -28,6 +28,7 @@ export async function collectAndPersistReports(
 
   const report = await collect(options);
   const sortedScoredReport = sortReport(scoreReport(report));
+
   const persistResults = await persistReport(
     report,
     sortedScoredReport,

packages/core/src/lib/implementation/runner.ts

@@ -4,7 +4,12 @@ import type {
   RunnerConfig,
   RunnerFunction,
 } from '@code-pushup/models';
-import { calcDuration, executeProcess, readJsonFile } from '@code-pushup/utils';
+import {
+  calcDuration,
+  executeProcess,
+  readJsonFile,
+  removeDirectoryIfExists,
+} from '@code-pushup/utils';
 
 export type RunnerResult = {
   date: string;
@@ -26,7 +31,9 @@ export async function executeRunnerConfig(
   });
 
   // read process output from file system and parse it
-  const outputs = await readJsonFile(path.join(process.cwd(), outputFile));
+  const outputs = await readJsonFile(outputFile);
+  // clean up plugin individual runner output directory
+  await removeDirectoryIfExists(path.dirname(outputFile));
 
   // transform unknownAuditOutputs to auditOutputs
   const audits = outputTransform ? await outputTransform(outputs) : outputs;

packages/models/src/index.ts

@@ -99,9 +99,11 @@ export {
   onProgressSchema,
   runnerConfigSchema,
   runnerFunctionSchema,
+  runnerFilesPathsSchema,
   type OnProgress,
   type RunnerConfig,
   type RunnerFunction,
+  type RunnerFilesPaths,
 } from './lib/runner-config.js';
 export {
   tableAlignmentSchema,

packages/models/src/lib/runner-config.ts

@@ -15,8 +15,9 @@
       description: 'Shell command to execute',
     }),
     args: z.array(z.string({ description: 'Command arguments' })).optional(),
-    outputFile: filePathSchema.describe('Output path'),
+    outputFile: filePathSchema.describe('Runner output path'),
     outputTransform: outputTransformSchema.optional(),
+    configFile: filePathSchema.describe('Runner config path').optional(),
   },
   {
     description: 'How to execute runner',
@@ -37,3 +38,10 @@
   .returns(z.union([auditOutputsSchema, z.promise(auditOutputsSchema)]));
 
 export type RunnerFunction = z.infer<typeof runnerFunctionSchema>;
+
+export const runnerFilesPathsSchema = z.object({
+  runnerConfigPath: filePathSchema.describe('Runner config path'),
+  runnerOutputPath: filePathSchema.describe('Runner output path'),
+});
+
+export type RunnerFilesPaths = z.infer<typeof runnerFilesPathsSchema>;

packages/nx-plugin/src/executors/cli/executor.unit.test.ts

@@ -2,6 +2,7 @@ import { logger } from '@nx/devkit';
 import { execSync } from 'node:child_process';
 import { afterEach, beforeEach, expect, vi } from 'vitest';
 import { executorContext } from '@code-pushup/test-nx-utils';
+import { MEMFS_VOLUME } from '@code-pushup/test-utils';
 import runAutorunExecutor from './executor.js';
 
 vi.mock('node:child_process', async () => {
@@ -39,7 +40,7 @@ describe('runAutorunExecutor', () => {
     // eslint-disable-next-line n/no-sync
     expect(execSync).toHaveBeenCalledWith(
       expect.stringContaining('npx @code-pushup/cli'),
-      { cwd: '/test' },
+      { cwd: MEMFS_VOLUME },
     );
   });
 

packages/plugin-coverage/package.json

@@ -38,6 +38,7 @@
     "@code-pushup/utils": "0.57.0",
     "ansis": "^3.3.0",
     "parse-lcov": "^1.0.4",
+    "yargs": "^17.7.2",
     "zod": "^3.22.4"
   },
   "peerDependencies": {

packages/plugin-coverage/src/bin.ts

@@ -1,3 +1,7 @@
+import process from 'node:process';
+import { Parser } from 'yargs/helpers';
 import { executeRunner } from './lib/runner/index.js';
 
-await executeRunner();
+const { runnerConfigPath, runnerOutputPath } = Parser(process.argv);
+
+await executeRunner({ runnerConfigPath, runnerOutputPath });

packages/plugin-coverage/src/lib/runner/constants.ts

@@ -1,12 +1 @@
-import path from 'node:path';
-import { pluginWorkDir } from '@code-pushup/utils';
-
-export const WORKDIR = pluginWorkDir('coverage');
-export const RUNNER_OUTPUT_PATH = path.join(WORKDIR, 'runner-output.json');
-export const PLUGIN_CONFIG_PATH = path.join(
-  process.cwd(),
-  WORKDIR,
-  'plugin-config.json',
-);
-
 export const INVALID_FUNCTION_NAME = '(empty-report)';

packages/plugin-coverage/src/lib/runner/index.ts

@@ -1,23 +1,31 @@
 import { bold } from 'ansis';
 import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
-import type { AuditOutputs, RunnerConfig } from '@code-pushup/models';
+import type {
+  AuditOutputs,
+  RunnerConfig,
+  RunnerFilesPaths,
+} from '@code-pushup/models';
 import {
   ProcessError,
+  createRunnerFiles,
   ensureDirectoryExists,
   executeProcess,
   filePathToCliArg,
+  objectToCliArgs,
   readJsonFile,
   ui,
 } from '@code-pushup/utils';
 import type { FinalCoveragePluginConfig } from '../config.js';
 import { applyMaxScoreAboveThreshold } from '../utils.js';
-import { PLUGIN_CONFIG_PATH, RUNNER_OUTPUT_PATH } from './constants.js';
 import { lcovResultsToAuditOutputs } from './lcov/lcov-runner.js';
 
-export async function executeRunner(): Promise<void> {
+export async function executeRunner({
+  runnerConfigPath,
+  runnerOutputPath,
+}: RunnerFilesPaths): Promise<void> {
   const { reports, coverageToolCommand, coverageTypes } =
-    await readJsonFile<FinalCoveragePluginConfig>(PLUGIN_CONFIG_PATH);
+    await readJsonFile<FinalCoveragePluginConfig>(runnerConfigPath);
 
   // Run coverage tool if provided
   if (coverageToolCommand != null) {
@@ -41,24 +49,30 @@ export async function executeRunner(): Promise<void> {
   // Calculate coverage from LCOV results
   const auditOutputs = await lcovResultsToAuditOutputs(reports, coverageTypes);
 
-  await ensureDirectoryExists(path.dirname(RUNNER_OUTPUT_PATH));
-  await writeFile(RUNNER_OUTPUT_PATH, JSON.stringify(auditOutputs));
+  await ensureDirectoryExists(path.dirname(runnerOutputPath));
+  await writeFile(runnerOutputPath, JSON.stringify(auditOutputs));
 }
 
 export async function createRunnerConfig(
   scriptPath: string,
   config: FinalCoveragePluginConfig,
 ): Promise<RunnerConfig> {
   // Create JSON config for executeRunner
-  await ensureDirectoryExists(path.dirname(PLUGIN_CONFIG_PATH));
-  await writeFile(PLUGIN_CONFIG_PATH, JSON.stringify(config));
+  const { runnerConfigPath, runnerOutputPath } = await createRunnerFiles(
+    'coverage',
+    JSON.stringify(config),
+  );
 
   const threshold = config.perfectScoreThreshold;
 
   return {
     command: 'node',
-    args: [filePathToCliArg(scriptPath)],
-    outputFile: RUNNER_OUTPUT_PATH,
+    args: [
+      filePathToCliArg(scriptPath),
+      ...objectToCliArgs({ runnerConfigPath, runnerOutputPath }),
+    ],
+    configFile: runnerConfigPath,
+    outputFile: runnerOutputPath,
     ...(threshold != null && {
       outputTransform: outputs =>
         applyMaxScoreAboveThreshold(outputs as AuditOutputs, threshold),

packages/plugin-coverage/src/lib/runner/runner.integration.test.ts

@@ -1,19 +1,13 @@
-import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { describe, it } from 'vitest';
+import { expect } from 'vitest';
 import type {
   AuditOutput,
   AuditOutputs,
   RunnerConfig,
 } from '@code-pushup/models';
-import { readJsonFile, removeDirectoryIfExists } from '@code-pushup/utils';
+import { createRunnerFiles, readJsonFile } from '@code-pushup/utils';
 import type { FinalCoveragePluginConfig } from '../config.js';
-import {
-  PLUGIN_CONFIG_PATH,
-  RUNNER_OUTPUT_PATH,
-  WORKDIR,
-} from './constants.js';
 import { createRunnerConfig, executeRunner } from './index.js';
 
 describe('createRunnerConfig', () => {
@@ -25,26 +19,32 @@ describe('createRunnerConfig', () => {
     });
     expect(runnerConfig).toStrictEqual<RunnerConfig>({
       command: 'node',
-      args: ['"executeRunner.ts"'],
+      args: [
+        '"executeRunner.ts"',
+        expect.stringContaining('plugin-config.json'),
+        expect.stringContaining('runner-output.json'),
+      ],
       outputTransform: expect.any(Function),
       outputFile: expect.stringContaining('runner-output.json'),
+      configFile: expect.stringContaining('plugin-config.json'),
     });
   });
 
   it('should provide plugin config to runner in JSON file', async () => {
-    await removeDirectoryIfExists(WORKDIR);
-
     const pluginConfig: FinalCoveragePluginConfig = {
       coverageTypes: ['line'],
       reports: ['coverage/lcov.info'],
       coverageToolCommand: { command: 'npm', args: ['run', 'test'] },
       perfectScoreThreshold: 85,
     };
 
-    await createRunnerConfig('executeRunner.ts', pluginConfig);
+    const { configFile } = await createRunnerConfig(
+      'executeRunner.ts',
+      pluginConfig,
+    );
 
-    const config =
-      await readJsonFile<FinalCoveragePluginConfig>(PLUGIN_CONFIG_PATH);
+    expect(configFile).toMatch(/.*plugin-config\.json$/);
+    const config = await readJsonFile<FinalCoveragePluginConfig>(configFile!);
     expect(config).toStrictEqual(pluginConfig);
   });
 });
@@ -65,10 +65,15 @@ describe('executeRunner', () => {
       coverageTypes: ['line'],
     };
 
-    await writeFile(PLUGIN_CONFIG_PATH, JSON.stringify(config));
-    await executeRunner();
+    const runnerFiles = await createRunnerFiles(
+      'coverage',
+      JSON.stringify(config),
+    );
+    await executeRunner(runnerFiles);
 
-    const results = await readJsonFile<AuditOutputs>(RUNNER_OUTPUT_PATH);
+    const results = await readJsonFile<AuditOutputs>(
+      runnerFiles.runnerOutputPath,
+    );
     expect(results).toStrictEqual([
       expect.objectContaining({
         slug: 'line-coverage',

packages/plugin-eslint/package.json

@@ -40,6 +40,7 @@
   "dependencies": {
     "@code-pushup/utils": "0.57.0",
     "@code-pushup/models": "0.57.0",
+    "yargs": "^17.7.2",
     "zod": "^3.22.4"
   },
   "peerDependencies": {

packages/plugin-eslint/src/bin.ts

@@ -1,3 +1,7 @@
+import process from 'node:process';
+import { Parser } from 'yargs/helpers';
 import { executeRunner } from './lib/runner/index.js';
 
-await executeRunner();
+const { runnerConfigPath, runnerOutputPath } = Parser(process.argv);
+
+await executeRunner({ runnerConfigPath, runnerOutputPath });

packages/plugin-eslint/src/lib/__snapshots__/eslint-plugin.integration.test.ts.snap

@@ -349,9 +349,12 @@ exports[`eslintPlugin > should initialize ESLint plugin for React application 1`
   "runner": {
     "args": [
       ""<dirname>/bin.js"",
+      "--runnerConfigPath="node_modules/.code-pushup/eslint/<timestamp>/plugin-config.json"",
+      "--runnerOutputPath="node_modules/.code-pushup/eslint/<timestamp>/runner-output.json"",
     ],
     "command": "node",
-    "outputFile": "node_modules/.code-pushup/eslint/runner-output.json",
+    "configFile": "node_modules/.code-pushup/eslint/<timestamp>/plugin-config.json",
+    "outputFile": "node_modules/.code-pushup/eslint/<timestamp>/runner-output.json",
   },
   "slug": "eslint",
   "title": "ESLint",

packages/plugin-eslint/src/lib/eslint-plugin.integration.test.ts

@@ -1,5 +1,6 @@
 import os from 'node:os';
 import path from 'node:path';
+import process from 'node:process';
 import { fileURLToPath } from 'node:url';
 import type { MockInstance } from 'vitest';
 import type { Audit, PluginConfig, RunnerConfig } from '@code-pushup/models';
@@ -19,9 +20,19 @@ describe('eslintPlugin', () => {
     runner: {
       ...(plugin.runner as RunnerConfig),
       args: (plugin.runner as RunnerConfig).args?.map(arg =>
-        toUnixPath(arg.replace(path.dirname(thisDir), '<dirname>')),
+        toUnixPath(arg.replace(path.dirname(thisDir), '<dirname>')).replace(
+          /\/eslint\/\d+\//,
+          '/eslint/<timestamp>/',
+        ),
       ),
-      outputFile: toUnixPath((plugin.runner as RunnerConfig).outputFile),
+      ...((plugin.runner as RunnerConfig).configFile && {
+        configFile: toUnixPath(
+          (plugin.runner as RunnerConfig).configFile!,
+        ).replace(/\/eslint\/\d+\//, '/eslint/<timestamp>/'),
+      }),
+      outputFile: toUnixPath(
+        (plugin.runner as RunnerConfig).outputFile,
+      ).replace(/\/eslint\/\d+\//, '/eslint/<timestamp>/'),
     },
   });
 
@@ -38,6 +49,7 @@ describe('eslintPlugin', () => {
 
   it('should initialize ESLint plugin for React application', async () => {
     cwdSpy.mockReturnValue(path.join(fixturesDir, 'todos-app'));
+
     const plugin = await eslintPlugin({
       eslintrc: 'eslint.config.js',
       patterns: ['src/**/*.js', 'src/**/*.jsx'],

packages/plugin-eslint/src/lib/meta/index.ts

@@ -13,7 +13,7 @@ export { detectConfigVersion, type ConfigFormat } from './versions/index.js';
 
 export async function listAuditsAndGroups(
   targets: ESLintTarget[],
-  customGroups: CustomGroup[] | undefined,
+  customGroups?: CustomGroup[] | undefined,
 ): Promise<{ audits: Audit[]; groups: Group[] }> {
   const rules = await listRules(targets);
 

packages/plugin-eslint/src/lib/nx.integration.test.ts

@@ -1,4 +1,5 @@
 import path from 'node:path';
+import process from 'node:process';
 import { fileURLToPath } from 'node:url';
 import type { MockInstance } from 'vitest';
 import { executeProcess } from '@code-pushup/utils';
@@ -11,7 +12,8 @@ import {
 
 type Project = 'cli' | 'core' | 'nx-plugin' | 'utils';
 
-describe('Nx helpers', () => {
+// skipping tests on Windows due to a problem with createProjectGraphAsync that hangs forever, issue seems to be connected to nested git or some other Nx graph related problem https://github.com/nrwl/nx/issues/27494#issuecomment-2633836688
+describe.skipIf(process.platform === 'win32')('Nx helpers', () => {
   let cwdSpy: MockInstance<[], string>;
 
   beforeAll(async () => {