diff --git a/packages/core/src/lib/implementation/execute-plugin.ts b/packages/core/src/lib/implementation/execute-plugin.ts index a08b5f924..b142c5ba0 100644 --- a/packages/core/src/lib/implementation/execute-plugin.ts +++ b/packages/core/src/lib/implementation/execute-plugin.ts @@ -10,9 +10,11 @@ import { auditOutputsSchema, } from '@code-pushup/models'; import { + ProgressBar, getProgressBar, groupByStatus, logMultipleResults, + pluralizeToken, } from '@code-pushup/utils'; import { normalizeAuditOutputs } from '../normalize'; import { executeRunnerConfig, executeRunnerFunction } from './runner'; @@ -22,7 +24,11 @@ import { executeRunnerConfig, executeRunnerFunction } from './runner'; */ export class PluginOutputMissingAuditError extends Error { constructor(auditSlug: string) { - super(`Audit metadata not found for slug ${auditSlug}`); + super( + `Audit metadata not present in plugin config. Missing slug: ${chalk.bold( + auditSlug, + )}`, + ); } } @@ -69,7 +75,11 @@ export async function executePlugin( const { audits: unvalidatedAuditOutputs, ...executionMeta } = runnerResult; // validate auditOutputs - const auditOutputs = auditOutputsSchema.parse(unvalidatedAuditOutputs); + const result = auditOutputsSchema.safeParse(unvalidatedAuditOutputs); + if (!result.success) { + throw new Error(`Audit output is invalid: ${result.error.message}`); + } + const auditOutputs = result.data; auditOutputsCorrelateWithPluginOutput(auditOutputs, pluginConfigAudits); const normalizedAuditOutputs = await normalizeAuditOutputs(auditOutputs); @@ -95,6 +105,28 @@ export async function executePlugin( }; } +const wrapProgress = async ( + pluginCfg: PluginConfig, + steps: number, + progressBar: ProgressBar | null, +) => { + progressBar?.updateTitle(`Executing ${chalk.bold(pluginCfg.title)}`); + try { + const pluginReport = await executePlugin(pluginCfg); + progressBar?.incrementInSteps(steps); + return pluginReport; + } catch (error) { + progressBar?.incrementInSteps(steps); + throw new Error( + error instanceof Error + ? `- Plugin ${chalk.bold(pluginCfg.title)} (${chalk.bold( + pluginCfg.slug, + )}) produced the following error:\n - ${error.message}` + : String(error), + ); + } +}; + /** * Execute multiple plugins and aggregates their output. * @public @@ -124,21 +156,13 @@ export async function executePlugins( const progressBar = progress ? getProgressBar('Run plugins') : null; - const pluginsResult = await plugins.reduce(async (acc, pluginCfg) => { - progressBar?.updateTitle(`Executing ${chalk.bold(pluginCfg.title)}`); - - try { - const pluginReport = await executePlugin(pluginCfg); - progressBar?.incrementInSteps(plugins.length); - return [...(await acc), Promise.resolve(pluginReport)]; - } catch (error) { - progressBar?.incrementInSteps(plugins.length); - return [ - ...(await acc), - Promise.reject(error instanceof Error ? error.message : String(error)), - ]; - } - }, Promise.resolve([] as Promise[])); + const pluginsResult = await plugins.reduce( + async (acc, pluginCfg) => [ + ...(await acc), + wrapProgress(pluginCfg, plugins.length, progressBar), + ], + Promise.resolve([] as Promise[]), + ); progressBar?.endProgress('Done running plugins'); @@ -151,9 +175,12 @@ export async function executePlugins( if (rejected.length > 0) { const errorMessages = rejected .map(({ reason }) => String(reason)) - .join(', '); + .join('\n'); throw new Error( - `Plugins failed: ${rejected.length} errors: ${errorMessages}`, + `Executing ${pluralizeToken( + 'plugin', + rejected.length, + )} failed.\n\n${errorMessages}\n\n`, ); } diff --git a/packages/core/src/lib/implementation/execute-plugin.unit.test.ts b/packages/core/src/lib/implementation/execute-plugin.unit.test.ts index f28aaa265..35d3087cd 100644 --- a/packages/core/src/lib/implementation/execute-plugin.unit.test.ts +++ b/packages/core/src/lib/implementation/execute-plugin.unit.test.ts @@ -1,12 +1,11 @@ +import chalk from 'chalk'; import { vol } from 'memfs'; -import { describe, expect, it, vi } from 'vitest'; +import { describe, expect, it } from 'vitest'; import { AuditOutputs, PluginConfig } from '@code-pushup/models'; import { MEMFS_VOLUME, MINIMAL_PLUGIN_CONFIG_MOCK, - getLogMessages, } from '@code-pushup/test-utils'; -import { ui } from '@code-pushup/utils'; import { PluginOutputMissingAuditError, executePlugin, @@ -65,7 +64,7 @@ describe('executePlugin', () => { ]); }); - it('should throw when plugin slug is invalid', async () => { + it('should throw when audit slug is invalid', async () => { await expect(() => executePlugin({ ...MINIMAL_PLUGIN_CONFIG_MOCK, @@ -74,19 +73,24 @@ describe('executePlugin', () => { ).rejects.toThrow(new PluginOutputMissingAuditError('node-version')); }); - it('should throw if invalid runnerOutput is produced', async () => { + it('should throw for missing audit', async () => { + const missingSlug = 'missing-audit-slug'; await expect(() => executePlugin({ ...MINIMAL_PLUGIN_CONFIG_MOCK, runner: () => [ { - slug: '-invalid-audit-slug', + slug: missingSlug, score: 0, value: 0, }, ], }), - ).rejects.toThrow('The slug has to follow the pattern'); + ).rejects.toThrow( + `Audit metadata not present in plugin config. Missing slug: ${chalk.bold( + missingSlug, + )}`, + ); }); }); @@ -108,55 +112,153 @@ describe('executePlugins', () => { expect(pluginResult[0]?.audits[0]?.slug).toBe('node-version'); }); - it('should throw for invalid plugins', async () => { + it('should throw for invalid audit output', async () => { + const slug = 'simulate-invalid-audit-slug'; + const title = 'Simulate an invalid audit slug in outputs'; await expect(() => executePlugins( [ - MINIMAL_PLUGIN_CONFIG_MOCK, { ...MINIMAL_PLUGIN_CONFIG_MOCK, - audits: [{ slug: '-invalid-slug', title: 'Invalid audit' }], + slug, + title, + runner: () => [ + { + slug: 'invalid-audit-slug-', + score: 0.3, + value: 16, + displayValue: '16.0.0', + }, + ], }, ] satisfies PluginConfig[], { progress: false }, ), - ).rejects.toThrow( - /Plugins failed: 1 errors:.*Audit metadata not found for slug node-version/, - ); + ).rejects + .toThrow(`Executing 1 plugin failed.\n\nError: - Plugin ${chalk.bold( + title, + )} (${chalk.bold(slug)}) produced the following error: + - Audit output is invalid: [ + { + "validation": "regex", + "code": "invalid_string", + "message": "The slug has to follow the pattern [0-9a-z] followed by multiple optional groups of -[0-9a-z]. e.g. my-slug", + "path": [ + 0, + "slug" + ] + } +] +`); }); - it('should print invalid plugin errors and throw', async () => { - const pluginConfig = { - ...MINIMAL_PLUGIN_CONFIG_MOCK, - runner: vi - .fn() - .mockRejectedValue('Audit metadata not found for slug node-version'), - }; - const pluginConfig2 = { - ...MINIMAL_PLUGIN_CONFIG_MOCK, - runner: vi.fn().mockResolvedValue([]), - }; - const pluginConfig3 = { - ...MINIMAL_PLUGIN_CONFIG_MOCK, - runner: vi.fn().mockRejectedValue('plugin 3 error'), - }; + it('should throw for one failing plugin', async () => { + const missingAuditSlug = 'missing-audit-slug'; + await expect(() => + executePlugins( + [ + { + ...MINIMAL_PLUGIN_CONFIG_MOCK, + slug: 'plg1', + title: 'plg1', + runner: () => [ + { + slug: `${missingAuditSlug}-a`, + score: 0.3, + value: 16, + displayValue: '16.0.0', + }, + ], + }, + ] satisfies PluginConfig[], + { progress: false }, + ), + ).rejects.toThrow('Executing 1 plugin failed.\n\n'); + }); + it('should throw for multiple failing plugins', async () => { + const missingAuditSlug = 'missing-audit-slug'; await expect(() => - executePlugins([pluginConfig, pluginConfig2, pluginConfig3], { - progress: false, - }), + executePlugins( + [ + { + ...MINIMAL_PLUGIN_CONFIG_MOCK, + slug: 'plg1', + title: 'plg1', + runner: () => [ + { + slug: `${missingAuditSlug}-a`, + score: 0.3, + value: 16, + displayValue: '16.0.0', + }, + ], + }, + { + ...MINIMAL_PLUGIN_CONFIG_MOCK, + slug: 'plg2', + title: 'plg2', + runner: () => [ + { + slug: `${missingAuditSlug}-b`, + score: 0.3, + value: 16, + displayValue: '16.0.0', + }, + ], + }, + ] satisfies PluginConfig[], + { progress: false }, + ), + ).rejects.toThrow('Executing 2 plugins failed.\n\n'); + }); + + it('should throw with indentation in message', async () => { + const missingAuditSlug = 'missing-audit-slug'; + + await expect(() => + executePlugins( + [ + { + ...MINIMAL_PLUGIN_CONFIG_MOCK, + slug: 'plg1', + title: 'plg1', + runner: () => [ + { + slug: `${missingAuditSlug}-a`, + score: 0.3, + value: 16, + displayValue: '16.0.0', + }, + ], + }, + { + ...MINIMAL_PLUGIN_CONFIG_MOCK, + slug: 'plg2', + title: 'plg2', + runner: () => [ + { + slug: `${missingAuditSlug}-b`, + score: 0.3, + value: 16, + displayValue: '16.0.0', + }, + ], + }, + ] satisfies PluginConfig[], + { progress: false }, + ), ).rejects.toThrow( - 'Plugins failed: 2 errors: Audit metadata not found for slug node-version, plugin 3 error', + `Error: - Plugin ${chalk.bold('plg1')} (${chalk.bold( + 'plg1', + )}) produced the following error:\n - Audit metadata not present in plugin config. Missing slug: ${chalk.bold( + 'missing-audit-slug-a', + )}\nError: - Plugin ${chalk.bold('plg2')} (${chalk.bold( + 'plg2', + )}) produced the following error:\n - Audit metadata not present in plugin config. Missing slug: ${chalk.bold( + 'missing-audit-slug-b', + )}`, ); - const logs = getLogMessages(ui().logger); - expect(logs[0]).toBe('[ yellow(warn) ] Plugins failed: '); - expect(logs[1]).toBe( - '[ yellow(warn) ] Audit metadata not found for slug node-version', - ); - - expect(pluginConfig.runner).toHaveBeenCalled(); - expect(pluginConfig2.runner).toHaveBeenCalled(); - expect(pluginConfig3.runner).toHaveBeenCalled(); }); it('should use outputTransform if provided', async () => {