diff --git a/.dockerignore b/.dockerignore index 2ceb847c93..1ca391a43a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -34,10 +34,11 @@ npm-debug.log # exceptions !/bin/copy-dist.ts -!/bin/electron-forge/sign-windows.cjs +!/bin/cleanupNodeModules.ts # temporary exception to make copy-dist inside Docker build not fail -# TriliumNextTODO: make copy-dist *not* requiring to copy this file for builds other than electron-forge +# TriliumNextTODO: make copy-dist *not* requiring to copy these file for builds other than electron-forge !forge.config.cjs !/bin/tpl -!/bin/electron-forge/desktop.ejs \ No newline at end of file +!/bin/electron-forge/desktop.ejs +!/bin/electron-forge/sign-windows.cjs \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 4aac3160fd..635a44d3be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,8 +39,11 @@ COPY --from=builder /usr/src/app ./ RUN sed -i "/electron/d" package.json && \ npm ci --omit=dev && \ + node --experimental-strip-types ./bin/cleanupNodeModules.ts . --skip-prune-dev-deps && \ npm cache clean --force && \ - rm -rf /tmp/node-compile-cache + rm -rf \ + /tmp/node-compile-cache \ + /usr/src/app/bin/cleanupNodeModules.ts # Configure container EXPOSE 8080 diff --git a/Dockerfile.alpine b/Dockerfile.alpine index f837893999..88a4507238 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -34,8 +34,11 @@ COPY --from=builder /usr/src/app ./ RUN sed -i "/electron/d" package.json && \ npm ci --omit=dev && \ + node --experimental-strip-types ./bin/cleanupNodeModules.ts . --skip-prune-dev-deps && \ npm cache clean --force && \ - rm -rf /tmp/node-compile-cache + rm -rf \ + /tmp/node-compile-cache \ + /usr/src/app/bin/cleanupNodeModules.ts # Add application user RUN adduser -s /bin/false node; exit 0 diff --git a/bin/build-server.sh b/bin/build-server.sh index 052a8131bc..6677d7c19c 100755 --- a/bin/build-server.sh +++ b/bin/build-server.sh @@ -25,8 +25,16 @@ NODE_VERSION=22.14.0 BUILD_DIR="./build" DIST_DIR="./dist" +CLEANUP_SCRIPT="./bin/cleanupNodeModules.ts" -./bin/copy-trilium.sh + +# Trigger the build +echo "Build start" +npm run build:prepare-dist +echo "Build finished" + +# pruning of unnecessary files and devDeps in node_modules +node --experimental-strip-types $CLEANUP_SCRIPT $BUILD_DIR NODE_FILENAME=node-v${NODE_VERSION}-linux-${ARCH} diff --git a/bin/cleanupNodeModules.ts b/bin/cleanupNodeModules.ts new file mode 100644 index 0000000000..bb3e903316 --- /dev/null +++ b/bin/cleanupNodeModules.ts @@ -0,0 +1,109 @@ +import fs from "fs-extra"; +import path from "path"; +import type { Dirent } from "fs-extra"; +import { execSync } from "node:child_process"; + +/** + * Example usage with node >= v22: + * node --experimental-strip-types bin/cleanupNodeModules.ts /path/to/build/folder [--skip-prune-dev-deps] + * Example usage with tsx: + * tsx bin/cleanupNodeModules.ts /path/to/build/folder [--skip-prune-dev-deps] + */ +function main() { + + if (process.argv.length > 4 || process.argv.length < 3) { + console.error("Usage: cleanupNodeModules.ts [path-to-build-folder] [--skip-prune-dev-deps]"); + process.exit(1); + } + + const basePath = process.argv[2]; + const pruneDevDeps = process.argv[3] !== "--skip-prune-dev-deps"; + + if (!fs.existsSync(basePath)) { + console.error(`Supplied path '${basePath}' does not exist. Aborting.`); + process.exit(1); + } + + console.log(`Starting pruning of node_modules ${!pruneDevDeps ? '(skipping npm pruning)' : ''} in '${basePath}'...`); + cleanupNodeModules(basePath, pruneDevDeps); + console.log("Successfully pruned node_modules."); +} + +function cleanupNodeModules(basePath: string, pruneDevDeps: boolean = true) { + + // This needs to run for the server and Docker build, + // but needs to be skipped for electron-forge: its + // built-in pruning takes care of it already + if (pruneDevDeps) { + execSync(`npm ci --omit=dev --prefix ${basePath}`); + } + + const nodeModulesDirPath = path.join(basePath, "node_modules"); + const nodeModulesContent = fs.readdirSync(nodeModulesDirPath, { recursive: true, withFileTypes: true }); + //const libDir = fs.readdirSync(path.join(basePath, "./libraries"), { recursive: true, withFileTypes: true }); + + /** + * Delete unnecessary folders + */ + const filterableDirs = new Set([ + "demo", + "demos", + "doc", + "docs", + "example", + "examples", + "test", + "tests" + ]); + + nodeModulesContent + .filter(el => el.isDirectory() && filterableDirs.has(el.name)) + .forEach(dir => removeDirent(dir)); + + /** + * Delete unnecessary files based on file extension + * TODO filter out useless (README).md files + */ + const filterableFileExt = new Set([ + "ts", + "map" + ]); + + nodeModulesContent + // TriliumNextTODO: check if we can improve this naive file ext matching, without introducing any additional dependency + .filter(el => el.isFile() && filterableFileExt.has(el.name.split(".").at(-1) || "")) + .forEach(dir => removeDirent(dir)); + + + /** + * Delete specific unnecessary folders + * TODO: check if we want removeSync to throw an error, if path does not exist anymore -> currently it will silently fail + */ + const extraFoldersDelete = new Set([ + path.join(nodeModulesDirPath, ".bin"), + path.join(nodeModulesDirPath, "@excalidraw", "excalidraw", "dist", "dev"), + path.join(nodeModulesDirPath, "boxicons", "svg"), + path.join(nodeModulesDirPath, "boxicons", "node_modules"), + path.join(nodeModulesDirPath, "boxicons", "src"), + path.join(nodeModulesDirPath, "boxicons", "iconjar"), + path.join(nodeModulesDirPath, "@jimp", "plugin-print", "fonts"), + path.join(nodeModulesDirPath, "jimp", "dist", "browser") // missing "@" in front of jimp is not a typo here + ]); + + nodeModulesContent + .filter(el => el.isDirectory() && extraFoldersDelete.has(path.join(el.parentPath, el.name))) + .forEach(dir => removeDirent(dir)) +} + + +function removeDirent(el: Dirent) { + const elementToDelete = path.join(el.parentPath, el.name); + fs.removeSync(elementToDelete); + + if (process.env.VERBOSE) { + console.log(`Deleted ${elementToDelete}`); + } + +} + +main() \ No newline at end of file diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index eaa79808eb..404cf09912 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -11,16 +11,10 @@ function log(...args: any[]) { } } -function copyNodeModuleFileOrFolder(source: string) { - const destination = path.join(DEST_DIR, source); - log(`Copying ${source} to ${destination}`); - fs.ensureDirSync(path.dirname(destination)); - fs.copySync(source, destination); -} - try { const assetsToCopy = new Set([ + // copy node_module, to avoid downloading packages a 2nd time during pruning "./node_modules", "./images", "./libraries", @@ -33,6 +27,7 @@ try { "./README.md", "./forge.config.cjs", "./bin/tpl/", + "./bin/cleanupNodeModules.ts", "./bin/electron-forge/desktop.ejs", "./bin/electron-forge/sign-windows.cjs", "./src/views/", @@ -61,50 +56,10 @@ try { fs.copySync(dir, path.join(PUBLIC_DIR, path.basename(dir))); } - const nodeModulesFile = new Set([ - "node_modules/react/umd/react.production.min.js", - "node_modules/react/umd/react.development.js", - "node_modules/react-dom/umd/react-dom.production.min.js", - "node_modules/react-dom/umd/react-dom.development.js", - "node_modules/katex/dist/katex.min.js", - "node_modules/katex/dist/contrib/mhchem.min.js", - "node_modules/katex/dist/contrib/auto-render.min.js", - "node_modules/@highlightjs/cdn-assets/highlight.min.js", - ]); - - const nodeModulesFolder = new Set([ - "node_modules/@excalidraw/excalidraw/dist/prod/fonts/", - "node_modules/katex/dist/", - "node_modules/dayjs/", - "node_modules/boxicons/css/", - "node_modules/boxicons/fonts/", - "node_modules/jquery/dist/", - "node_modules/jquery-hotkeys/", - "node_modules/split.js/dist/", - "node_modules/i18next/", - "node_modules/i18next-http-backend/", - "node_modules/vanilla-js-wheel-zoom/dist/", - "node_modules/mark.js/dist/", - "node_modules/normalize.css/", - "node_modules/jquery.fancytree/dist/", - "node_modules/autocomplete.js/dist/", - "node_modules/codemirror/lib/", - "node_modules/codemirror/addon/", - "node_modules/codemirror/mode/", - "node_modules/codemirror/keymap/", - "node_modules/@highlightjs/cdn-assets/languages", - "node_modules/@highlightjs/cdn-assets/styles", - "node_modules/leaflet/dist" - ]); - - - - for (const nodeModuleItem of [...nodeModulesFile, ...nodeModulesFolder]) { - copyNodeModuleFileOrFolder(nodeModuleItem); - } console.log("Copying complete!") } catch(err) { console.error("Error during copy:", err) process.exit(1) } + diff --git a/bin/copy-trilium.sh b/bin/copy-trilium.sh deleted file mode 100755 index 55c5cf6de1..0000000000 --- a/bin/copy-trilium.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -set -e # Fail on any command error -shopt -s globstar - -BUILD_DIR="./build" - -if ! [[ $(which npm) ]]; then - echo "Missing npm" - exit 1 -fi - -# Trigger the build -echo Build start -npm run build:prepare-dist -echo Build finished - -# Patch package.json main -sed -i 's|./dist/electron-main.js|electron-main.js|g' "$BUILD_DIR/package.json" - -# run in subshell (so we return to original dir) -(cd $BUILD_DIR && npm ci --omit=dev) - -if [[ -d "$BUILD_DIR"/node_modules ]]; then - # cleanup of useless files in dependencies - for d in 'image-q/demo' \ - '@excalidraw/excalidraw/dist/excalidraw-assets-dev' '@excalidraw/excalidraw/dist/excalidraw.development.js' '@excalidraw/excalidraw/dist/excalidraw-with-preact.development.js' \ - 'mermaid/dist/mermaid.js' \ - 'boxicons/svg' 'boxicons/node_modules/react'/* \ - '@jimp/plugin-print/fonts' 'jimp/browser' 'jimp/fonts'; do - [[ -e "$BUILD_DIR"/node_modules/"$d" ]] && rm -r "$BUILD_DIR"/node_modules/"$d" - done - - # delete all tests (there are often large images as test file for jimp etc.) - for d in 'test' 'docs' 'demo' 'example'; do - find "$BUILD_DIR"/node_modules -name "$d" -exec rm -rf {} + - done -fi - -find $BUILD_DIR/libraries -name "*.map" -type f -delete -find $BUILD_DIR/node_modules -name "*.map" -type f -delete -find $BUILD_DIR -name "*.ts" -type f -delete - -unset f d BUILD_DIR diff --git a/forge.config.cjs b/forge.config.cjs index ba3d8c3ffb..8d5b5d22eb 100644 --- a/forge.config.cjs +++ b/forge.config.cjs @@ -1,5 +1,6 @@ const path = require("path"); const fs = require("fs-extra"); +const { execSync } = require("child_process"); const APP_NAME = "TriliumNext Notes"; const BIN_PATH = path.normalize("./bin/electron-forge"); @@ -39,6 +40,22 @@ module.exports = { "translations/", "node_modules/@highlightjs/cdn-assets/styles" ], + afterPrune: [ + (buildPath, _electronVersion, _platform, _arch, callback) => { + // buildPath is a temporary directory that electron-packager creates - it's in the form of + // /tmp/electron-packager/tmp-SjJl0s/resources/app + try { + const cleanupNodeModulesScript = path.join(buildPath, "bin", "cleanupNodeModules.ts"); + // we don't have access to any devDeps like 'tsx' here, so use the built-in '--experimental-strip-types' flag instead + const command = `node --experimental-strip-types ${cleanupNodeModulesScript} "${buildPath}" --skip-prune-dev-deps`; + // execSync throws, if above returns any non-zero exit code + execSync(command); + callback() + } catch(err) { + callback(err) + } + } + ], afterComplete: [ (buildPath, _electronVersion, platform, _arch, callback) => { // Only move resources on non-macOS platforms diff --git a/package.json b/package.json index 528605a814..57141ffa8a 100644 --- a/package.json +++ b/package.json @@ -38,10 +38,9 @@ "electron:switch": "electron-rebuild", "docs:edit": "cross-env NODE_OPTIONS=\"--import tsx\" TRILIUM_DATA_DIR=./data-docs TRILIUM_ENV=dev TRILIUM_PORT=37741 electron ./electron-docs-main.ts .", "docs:edit-nix": "electron-rebuild --version 33.3.1 && cross-env NODE_OPTIONS=\"--import tsx\" TRILIUM_DATA_DIR=./data-docs TRILIUM_PORT=37741 TRILIUM_ENV=dev nix-shell -p electron_33 --run \"electron ./electron-docs-main.ts .\"", - "electron-forge:prepare": "npm run build:prepare-dist", - "electron-forge:start": "npm run electron-forge:prepare && cd ./build && electron-forge start", - "electron-forge:make": "npm run electron-forge:prepare && cross-env DEBUG=electron-windows-installer:* electron-forge make ./build", - "electron-forge:package": "npm run electron-forge:prepare && cd ./build && electron-forge package", + "electron-forge:start": "npm run build:prepare-dist && cd ./build && electron-forge start", + "electron-forge:make": "npm run build:prepare-dist && cross-env DEBUG=electron-windows-installer:* electron-forge make ./build", + "electron-forge:package": "npm run build:prepare-dist && cd ./build && electron-forge package", "docs:build-backend": "rimraf ./docs/backend_api && typedoc ./docs/backend_api src/becca/entities/*.ts src/services/backend_script_api.ts src/services/sql.ts", "docs:build-frontend": "rimraf ./docs/frontend_api && jsdoc -c jsdoc-conf.json -d ./docs/frontend_api src/public/app/entities/*.js src/public/app/services/frontend_script_api.js src/public/app/widgets/basic_widget.js src/public/app/widgets/note_context_aware_widget.js src/public/app/widgets/right_panel_widget.js", "docs:build": "npm run docs:build-backend && npm run docs:build-frontend", @@ -232,7 +231,6 @@ "lorem-ipsum": "2.0.8", "mind-elixir": "4.4.3", "mini-css-extract-plugin": "2.9.2", - "node-abi": "4.2.0", "nodemon": "3.1.9", "postcss-loader": "8.1.1", "prettier": "3.5.3",