Merge pull request #17 from 343dev/improvement/lossless

Disable parallel optimization of JPEG files in Lossless mode

Author: 343dev

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 9.1.0 (2024-10-16)
+
+- Disabled parallel optimization of JPEG files in Lossless mode.
+
+Guetzli uses a huge amount of RAM. In my case, when optimizing a file of about 30 MB, one process could use up to 12 GB of memory. If there are multiple files, parallel optimization with Guetzli consumes all available RAM, causing the system to use Swap, leading to slowdowns and freezes.
+
+For this reason I decided to disable parallel optimization of JPEG files in Lossless mode. Now, it will take more time but will have less impact on the OS performance.
+
+
 ## 9.0.2 (2024-10-08)
 
 - Fixed Guetzli install.

convert.js

@@ -49,59 +49,57 @@ export async function convert({ filePaths, config }) {
 
 	const totalSize = { before: 0, after: 0 };
 
-	const avifConfig = isLossless
-		? config?.avif?.lossless
-		: config?.avif?.lossy;
-	const webpConfig = isLossless
-		? config?.webp?.lossless
-		: config?.webp?.lossy;
-	const webpGifConfig = isLossless
-		? config?.webpGif?.lossless
-		: config?.webpGif?.lossy;
-
-	const tasksSimultaneousLimit = pLimit(os.cpus().length);
-	const tasksPromises = filePaths.reduce((accumulator, filePath) => {
-		if (shouldConvertToAvif) {
-			accumulator.push(
-				tasksSimultaneousLimit(
-					() => processFile({
-						filePath,
-						config: avifConfig || {},
-						progressBarContainer,
-						progressBar,
-						totalSize,
-						isForced,
-						format: 'AVIF',
-						processFunction: processAvif,
-					}),
-				),
-			);
-		}
-
-		if (shouldConvertToWebp) {
-			accumulator.push(
-				tasksSimultaneousLimit(
-					() => processFile({
-						filePath,
-						config: (path.extname(filePath.input).toLowerCase() === '.gif'
-							? webpGifConfig
-							: webpConfig)
-							|| {},
-						progressBarContainer,
-						progressBar,
-						totalSize,
-						isForced,
-						format: 'WebP',
-						processFunction: processWebp,
-					}),
-				),
-			);
-		}
-
-		return accumulator;
-	}, []);
+	const getConfig = format => config?.[format]?.[isLossless ? 'lossless' : 'lossy'];
+
+	const avifConfig = getConfig('avif');
+	const webpConfig = getConfig('webp');
+	const webpGifConfig = getConfig('webpGif');
+
+	const cpuCount = os.cpus().length;
+	const tasksSimultaneousLimit = pLimit(cpuCount);
+
+	await Promise.all(
+		filePaths.reduce((accumulator, filePath) => {
+			if (shouldConvertToAvif) {
+				accumulator.push(
+					tasksSimultaneousLimit(
+						() => processFile({
+							filePath,
+							config: avifConfig || {},
+							progressBarContainer,
+							progressBar,
+							totalSize,
+							isForced,
+							format: 'AVIF',
+							processFunction: processAvif,
+						}),
+					),
+				);
+			}
+
+			if (shouldConvertToWebp) {
+				const isGif = path.extname(filePath.input).toLowerCase() === '.gif';
+
+				accumulator.push(
+					tasksSimultaneousLimit(
+						() => processFile({
+							filePath,
+							config: (isGif ? webpGifConfig : webpConfig) || {},
+							progressBarContainer,
+							progressBar,
+							totalSize,
+							isForced,
+							format: 'WebP',
+							processFunction: processWebp,
+						}),
+					),
+				);
+			}
+
+			return accumulator;
+		}, []),
+	);
 
-	await Promise.all(tasksPromises);
 	progressBarContainer.update(); // Prevent logs lost. See: https://github.com/npkgz/cli-progress/issues/145#issuecomment-1859594159
 	progressBarContainer.stop();
 

optimize.js

@@ -42,28 +42,29 @@ export async function optimize({ filePaths, config }) {
 	const totalSize = { before: 0, after: 0 };
 
 	const cpuCount = os.cpus().length;
-	const tasksSimultaneousLimit = pLimit(
-		/*
-			Guetzli uses a large amount of memory and a significant amount of CPU time.
-			To reduce the processor load in lossless mode, we reduce the number
-			of simultaneous tasks by half.
-		 */
-		isLossless ? Math.round(cpuCount / 2) : cpuCount,
-	);
-	const tasksPromises = filePaths.map(
-		filePath => tasksSimultaneousLimit(
-			() => processFile({
+	const tasksSimultaneousLimit = pLimit(cpuCount);
+	const guetzliTasksSimultaneousLimit = pLimit(1); // Guetzli uses a large amount of memory and a significant amount of CPU time. To reduce system load, we only allow one instance of guetzli to run at the same time.
+
+	await Promise.all(
+		filePaths.map(filePath => {
+			const extension = path.extname(filePath.input).toLowerCase();
+			const isJpeg = extension === '.jpg' || extension === '.jpeg';
+
+			const limit = isJpeg && isLossless
+				? guetzliTasksSimultaneousLimit
+				: tasksSimultaneousLimit;
+
+			return limit(() => processFile({
 				filePath,
 				config,
 				progressBarContainer,
 				progressBar,
 				totalSize,
 				isLossless,
-			}),
-		),
+			}));
+		}),
 	);
 
-	await Promise.all(tasksPromises);
 	progressBarContainer.update(); // Prevent logs lost. See: https://github.com/npkgz/cli-progress/issues/145#issuecomment-1859594159
 	progressBarContainer.stop();
 

package-lock.json

@@ -1,6 +1,6 @@
 {
 	"name": "@343dev/optimizt",
-	"version": "9.0.3",
+	"version": "9.1.0",
 	"description": "CLI image optimization tool",
 	"keywords": [
 		"svg",