diff --git a/completions/bun-cli.json b/completions/bun-cli.json index 5bcf1c989fbd..b930ee4b2815 100644 --- a/completions/bun-cli.json +++ b/completions/bun-cli.json @@ -373,7 +373,7 @@ }, { "name": "ignore-scripts", - "description": "Skip lifecycle scripts in the project's package.json (dependency scripts are never run)", + "description": "Skip lifecycle scripts for all packages, including the project's package.json and trusted dependencies", "hasValue": false, "required": false, "multiple": false @@ -427,7 +427,7 @@ }, { "name": "network-concurrency", - "description": "Maximum number of concurrent network requests (default 48)", + "description": "Maximum number of concurrent network requests (default 64)", "hasValue": true, "valueType": "val", "required": false, @@ -694,7 +694,7 @@ }, { "name": "ignore-scripts", - "description": "Skip lifecycle scripts in the project's package.json (dependency scripts are never run)", + "description": "Skip lifecycle scripts for all packages, including the project's package.json and trusted dependencies", "hasValue": false, "required": false, "multiple": false @@ -748,7 +748,7 @@ }, { "name": "network-concurrency", - "description": "Maximum number of concurrent network requests (default 48)", + "description": "Maximum number of concurrent network requests (default 64)", "hasValue": true, "valueType": "val", "required": false, @@ -1025,7 +1025,7 @@ }, { "name": "ignore-scripts", - "description": "Skip lifecycle scripts in the project's package.json (dependency scripts are never run)", + "description": "Skip lifecycle scripts for all packages, including the project's package.json and trusted dependencies", "hasValue": false, "required": false, "multiple": false @@ -1079,7 +1079,7 @@ }, { "name": "network-concurrency", - "description": "Maximum number of concurrent network requests (default 48)", + "description": "Maximum number of concurrent network requests (default 64)", "hasValue": true, "valueType": "val", "required": false, @@ -1296,7 +1296,7 @@ }, { "name": "ignore-scripts", - "description": "Skip lifecycle scripts in the project's package.json (dependency scripts are never run)", + "description": "Skip lifecycle scripts for all packages, including the project's package.json and trusted dependencies", "hasValue": false, "required": false, "multiple": false @@ -1351,7 +1351,7 @@ }, { "name": "network-concurrency", - "description": "Maximum number of concurrent network requests (default 48)", + "description": "Maximum number of concurrent network requests (default 64)", "hasValue": true, "valueType": "val", "required": false, @@ -1411,6 +1411,14 @@ "required": false, "multiple": false }, + { + "name": "libc", + "description": "Override libc for optional dependencies (e.g., glibc, musl, * for all)", + "hasValue": true, + "valueType": "val", + "required": false, + "multiple": false + }, { "name": "help", "shortName": "h", @@ -1530,6 +1538,38 @@ "valueType": "val", "required": false, "multiple": false + }, + { + "name": "production", + "shortName": "p", + "description": "Skip packages that are only needed by devDependencies (alias: --prod)", + "hasValue": false, + "required": false, + "multiple": false + }, + { + "name": "omit", + "description": "Skip packages that are only needed by the given dependency types: dev, optional, or peer (repeatable)", + "hasValue": true, + "valueType": "val", + "required": false, + "multiple": false + }, + { + "name": "cwd", + "description": "Set a specific cwd", + "hasValue": true, + "valueType": "val", + "required": false, + "multiple": false + }, + { + "name": "help", + "shortName": "h", + "description": "Print this help menu", + "hasValue": false, + "required": false, + "multiple": false } ], "positionalArgs": [ @@ -1714,7 +1754,7 @@ }, { "name": "ignore-scripts", - "description": "Skip lifecycle scripts in the project's package.json (dependency scripts are never run)", + "description": "Skip lifecycle scripts for all packages, including the project's package.json and trusted dependencies", "hasValue": false, "required": false, "multiple": false @@ -1769,7 +1809,7 @@ }, { "name": "network-concurrency", - "description": "Maximum number of concurrent network requests (default 48)", + "description": "Maximum number of concurrent network requests (default 64)", "hasValue": true, "valueType": "val", "required": false, @@ -1829,6 +1869,14 @@ "required": false, "multiple": false }, + { + "name": "libc", + "description": "Override libc for optional dependencies (e.g., glibc, musl, * for all)", + "hasValue": true, + "valueType": "val", + "required": false, + "multiple": false + }, { "name": "help", "shortName": "h", @@ -1843,6 +1891,13 @@ "hasValue": false, "required": false, "multiple": false + }, + { + "name": "why", + "description": "Also list each version's dependents and the ranges they asked for", + "hasValue": false, + "required": false, + "multiple": false } ], "positionalArgs": [ @@ -1885,6 +1940,13 @@ "required": false, "multiple": false }, + { + "name": "check", + "description": "Exit with code 1 if node_modules has packages that can be removed, without deleting anything", + "hasValue": false, + "required": false, + "multiple": false + }, { "name": "os", "description": "Prune for a different operating system than the current one", @@ -1901,6 +1963,14 @@ "required": false, "multiple": false }, + { + "name": "libc", + "description": "Prune for a different libc than the current one", + "hasValue": true, + "valueType": "val", + "required": false, + "multiple": false + }, { "name": "linker", "description": "Prune a node_modules installed with the given linker (one of \"isolated\" or \"hoisted\")", @@ -1950,7 +2020,7 @@ "type": "string" } ], - "examples": ["bun prune", "bun prune --production", "bun prune --dry-run"], + "examples": ["bun prune", "bun prune --production", "bun prune --dry-run", "bun prune --check"], "usage": "Usage: bun prune [flags]", "documentationUrl": "https://bun.com/docs/pm/cli/prune.", "dynamicCompletions": {} @@ -2095,7 +2165,7 @@ }, { "name": "ignore-scripts", - "description": "Skip lifecycle scripts in the project's package.json (dependency scripts are never run)", + "description": "Skip lifecycle scripts for all packages, including the project's package.json and trusted dependencies", "hasValue": false, "required": false, "multiple": false @@ -2149,7 +2219,7 @@ }, { "name": "network-concurrency", - "description": "Maximum number of concurrent network requests (default 48)", + "description": "Maximum number of concurrent network requests (default 64)", "hasValue": true, "valueType": "val", "required": false, @@ -2269,14 +2339,14 @@ }, { "name": "no-save", - "description": "Don't update package.json or save a lockfile", + "description": "Don't update package.json or save a lockfile (the default)", "hasValue": false, "required": false, "multiple": false }, { "name": "save", - "description": "Save to package.json (true by default)", + "description": "Update package.json and save a lockfile (false by default)", "hasValue": false, "required": false, "multiple": false @@ -2378,7 +2448,7 @@ }, { "name": "ignore-scripts", - "description": "Skip lifecycle scripts in the project's package.json (dependency scripts are never run)", + "description": "Skip lifecycle scripts for all packages, including the project's package.json and trusted dependencies", "hasValue": false, "required": false, "multiple": false @@ -2432,7 +2502,7 @@ }, { "name": "network-concurrency", - "description": "Maximum number of concurrent network requests (default 48)", + "description": "Maximum number of concurrent network requests (default 64)", "hasValue": true, "valueType": "val", "required": false, @@ -2492,7 +2562,7 @@ "completionType": "package" } ], - "examples": ["bun link", "bun link "], + "examples": ["bun link", "bun link ", "bun link --save "], "usage": "Usage: bun link [flags] []", "documentationUrl": "https://bun.com/docs/cli/link.", "dynamicCompletions": {} @@ -2528,14 +2598,14 @@ }, { "name": "no-save", - "description": "Don't update package.json or save a lockfile", + "description": "Don't update package.json or save a lockfile (the default)", "hasValue": false, "required": false, "multiple": false }, { "name": "save", - "description": "Save to package.json (true by default)", + "description": "Update package.json and save a lockfile (false by default)", "hasValue": false, "required": false, "multiple": false @@ -2637,7 +2707,7 @@ }, { "name": "ignore-scripts", - "description": "Skip lifecycle scripts in the project's package.json (dependency scripts are never run)", + "description": "Skip lifecycle scripts for all packages, including the project's package.json and trusted dependencies", "hasValue": false, "required": false, "multiple": false @@ -2691,7 +2761,7 @@ }, { "name": "network-concurrency", - "description": "Maximum number of concurrent network requests (default 48)", + "description": "Maximum number of concurrent network requests (default 64)", "hasValue": true, "valueType": "val", "required": false, @@ -2889,7 +2959,7 @@ }, { "name": "ignore-scripts", - "description": "Skip lifecycle scripts in the project's package.json (dependency scripts are never run)", + "description": "Skip lifecycle scripts for all packages, including the project's package.json and trusted dependencies", "hasValue": false, "required": false, "multiple": false @@ -2943,7 +3013,7 @@ }, { "name": "network-concurrency", - "description": "Maximum number of concurrent network requests (default 48)", + "description": "Maximum number of concurrent network requests (default 64)", "hasValue": true, "valueType": "val", "required": false, @@ -3194,7 +3264,7 @@ }, { "name": "ignore-scripts", - "description": "Skip lifecycle scripts in the project's package.json (dependency scripts are never run)", + "description": "Skip lifecycle scripts for all packages, including the project's package.json and trusted dependencies", "hasValue": false, "required": false, "multiple": false @@ -3248,7 +3318,7 @@ }, { "name": "network-concurrency", - "description": "Maximum number of concurrent network requests (default 48)", + "description": "Maximum number of concurrent network requests (default 64)", "hasValue": true, "valueType": "val", "required": false, @@ -3368,7 +3438,7 @@ }, "ls": { "name": "ls", - "description": "list the dependency tree according to the current lockfile", + "description": "list the tree of installed dependencies", "flags": [], "positionalArgs": [] }, @@ -3623,7 +3693,7 @@ }, { "name": "ignore-scripts", - "description": "Skip lifecycle scripts in the project's package.json (dependency scripts are never run)", + "description": "Skip lifecycle scripts for all packages, including the project's package.json and trusted dependencies", "hasValue": false, "required": false, "multiple": false @@ -3677,7 +3747,7 @@ }, { "name": "network-concurrency", - "description": "Maximum number of concurrent network requests (default 48)", + "description": "Maximum number of concurrent network requests (default 64)", "hasValue": true, "valueType": "val", "required": false, diff --git a/completions/bun.bash b/completions/bun.bash index 29304d7e69ca..ab071e3aca25 100644 --- a/completions/bun.bash +++ b/completions/bun.bash @@ -104,8 +104,8 @@ _bun_completions() { PACKAGE_OPTIONS[SHARED_OPTIONS_LONG]="--config --yarn --production --frozen-lockfile --no-save --dry-run --force --cache-dir --no-cache --silent --verbose --global --cwd --backend --link-native-bins --help"; PACKAGE_OPTIONS[SHARED_OPTIONS_SHORT]="-c -y -p -f -g"; - PACKAGE_OPTIONS[DEDUPE_OPTIONS_LONG]="--check"; - PACKAGE_OPTIONS[PRUNE_OPTIONS_LONG]="--production --prod --omit --filter --dry-run --os --cpu --linker --silent --cwd --help"; + PACKAGE_OPTIONS[DEDUPE_OPTIONS_LONG]="--check --why"; + PACKAGE_OPTIONS[PRUNE_OPTIONS_LONG]="--production --prod --omit --filter --dry-run --check --os --cpu --libc --linker --silent --cwd --help"; PACKAGE_OPTIONS[PRUNE_OPTIONS_SHORT]="-p -P -F -h"; PACKAGE_OPTIONS[AUDIT_OPTIONS_LONG]="--json --audit-level --ignore --prod --production --omit --dry-run --latest --cwd --help"; PACKAGE_OPTIONS[AUDIT_OPTIONS_SHORT]="-L"; @@ -139,6 +139,9 @@ _bun_completions() { --linker) COMPREPLY=( $(compgen -W "isolated hoisted" -- "${cur_word}") ); return;; + --libc) + COMPREPLY=( $(compgen -W "glibc musl" -- "${cur_word}") ); + return;; --cwd|--public-dir) COMPREPLY=( $(compgen -d -- "${cur_word}" )); return;; diff --git a/completions/bun.fish b/completions/bun.fish index 5e94166f5ecb..9acf8bb292ab 100644 --- a/completions/bun.fish +++ b/completions/bun.fish @@ -166,6 +166,9 @@ complete -c bun \ complete -c bun \ -n "__fish_seen_subcommand_from dedupe" -l 'check' -d 'Exit with code 1 if the lockfile has duplicate versions that can be removed, without changing anything' +complete -c bun \ + -n "__fish_seen_subcommand_from dedupe" -l 'why' -d "Also list each version's dependents and the ranges they asked for" + complete -c bun \ -n "__fish_seen_subcommand_from add" -d 'Popular' -a '(__fish__get_bun_packages)' @@ -220,11 +223,13 @@ complete -c bun -n "__fish_seen_subcommand_from audit" -l "ignore" -r -d "Ignore complete -c bun -n "__fish_seen_subcommand_from audit" -l "prod" -d "Omit devDependencies" -f complete -c bun -n "__fish_seen_subcommand_from audit prune" -l "omit" -r -a "dev optional peer" -d "Omit the given dependency type" -f complete -c bun -n "__fish_seen_subcommand_from audit prune" -l "dry-run" -d "Print what would change without changing anything" -f +complete -c bun -n "__fish_seen_subcommand_from prune" -l "check" -d "Exit with code 1 if node_modules has packages that can be removed, without deleting anything" -f complete -c bun -n "__fish_seen_subcommand_from audit; and __fish_seen_subcommand_from fix" -s "L" -l "latest" -d "Also apply fixes that fall outside the ranges declared in package.json or catalogs" -f complete -c bun -n "__fish_seen_subcommand_from prune" -s "p" -l "production" -d "Also remove packages that are only needed by devDependencies" -f complete -c bun -n "__fish_seen_subcommand_from prune" -s "P" -l "prod" -d "Also remove packages that are only needed by devDependencies" -f complete -c bun -n "__fish_seen_subcommand_from prune" -l "os" -r -d "Prune for a different operating system than the current one" -f complete -c bun -n "__fish_seen_subcommand_from prune" -l "cpu" -r -d "Prune for a different CPU architecture than the current one" -f +complete -c bun -n "__fish_seen_subcommand_from prune" -l "libc" -r -a "glibc musl" -d "Prune for a different libc than the current one" -f complete -c bun -n "__fish_seen_subcommand_from prune" -l "linker" -r -a "isolated hoisted" -d "Prune a node_modules installed with the given linker" -f complete -c bun -n "__fish_seen_subcommand_from prune" -s "F" -l "filter" -r -d "Prune only the matching workspaces" -f complete -c bun -n "__fish_seen_subcommand_from prune" -l "silent" -d "Don't log anything" -f diff --git a/completions/bun.zsh b/completions/bun.zsh index 3a02d58b200c..22ee60492abf 100644 --- a/completions/bun.zsh +++ b/completions/bun.zsh @@ -23,7 +23,7 @@ _bun_add_completion() { '--no-progress[Disable the progress bar]' \ '--no-summary[Don'"'"'t print a summary]' \ '--no-verify[Skip verifying integrity of newly downloaded packages]' \ - '--ignore-scripts[Skip lifecycle scripts in the package.json (dependency scripts are never run)]' \ + '--ignore-scripts[Skip lifecycle scripts for all packages, including the project'"'"'s package.json and trusted dependencies]' \ '--global[Add a package globally]' \ '-g[Add a package globally]' \ '--cwd[Set a specific cwd]:cwd' \ @@ -77,7 +77,7 @@ _bun_unlink_completion() { '--no-progress[Disable the progress bar]' \ '--no-summary[Don'"'"'t print a summary]' \ '--no-verify[Skip verifying integrity of newly downloaded packages]' \ - '--ignore-scripts[Skip lifecycle scripts in the package.json (dependency scripts are never run)]' \ + '--ignore-scripts[Skip lifecycle scripts for all packages, including the project'"'"'s package.json and trusted dependencies]' \ '--global[Add a package globally]' \ '-g[Add a package globally]' \ '--cwd[Set a specific cwd]:cwd' \ @@ -121,7 +121,7 @@ _bun_link_completion() { '--no-progress[Disable the progress bar]' \ '--no-summary[Don'"'"'t print a summary]' \ '--no-verify[Skip verifying integrity of newly downloaded packages]' \ - '--ignore-scripts[Skip lifecycle scripts in the package.json (dependency scripts are never run)]' \ + '--ignore-scripts[Skip lifecycle scripts for all packages, including the project'"'"'s package.json and trusted dependencies]' \ '--global[Add a package globally]' \ '-g[Add a package globally]' \ '--cwd[Set a specific cwd]:cwd' \ @@ -259,7 +259,7 @@ _bun_pm_completion() { cmd2) sub_commands=( 'bin\:"print the path to bin folder" ' - 'ls\:"list the dependency tree according to the current lockfile" ' + 'ls\:"list the tree of installed dependencies" ' 'licenses\:"list installed packages grouped by license" ' 'hash\:"generate & print the hash of the current lockfile" ' 'hash-string\:"print the string used to hash the lockfile" ' @@ -295,7 +295,7 @@ _bun_pm_completion() { ;; ls) pmargs=( - "--all[list the entire dependency tree according to the current lockfile]" + "--all[list the entire tree of installed dependencies]" "--trusted[list only trusted dependencies]" ) @@ -387,7 +387,7 @@ _bun_install_completion() { '--no-progress[Disable the progress bar]' \ '--no-summary[Don'"'"'t print a summary]' \ '--no-verify[Skip verifying integrity of newly downloaded packages]' \ - '--ignore-scripts[Skip lifecycle scripts in the package.json (dependency scripts are never run)]' \ + '--ignore-scripts[Skip lifecycle scripts for all packages, including the project'"'"'s package.json and trusted dependencies]' \ '--global[Add a package globally]' \ '-g[Add a package globally]' \ '--cwd[Set a specific cwd]:cwd' \ @@ -437,7 +437,7 @@ _bun_remove_completion() { '--no-progress[Disable the progress bar]' \ '--no-summary[Don'"'"'t print a summary]' \ '--no-verify[Skip verifying integrity of newly downloaded packages]' \ - '--ignore-scripts[Skip lifecycle scripts in the package.json (dependency scripts are never run)]' \ + '--ignore-scripts[Skip lifecycle scripts for all packages, including the project'"'"'s package.json and trusted dependencies]' \ '--global[Add a package globally]' \ '-g[Add a package globally]' \ '--cwd[Set a specific cwd]:cwd' \ @@ -656,7 +656,7 @@ _bun_update_completion() { '--no-progress[Disable the progress bar]' \ '--no-summary[Don'"'"'t print a summary]' \ '--no-verify[Skip verifying integrity of newly downloaded packages]' \ - '--ignore-scripts[Skip lifecycle scripts in the package.json (dependency scripts are never run)]' \ + '--ignore-scripts[Skip lifecycle scripts for all packages, including the project'"'"'s package.json and trusted dependencies]' \ '-g[Add a package globally]' \ '--global[Add a package globally]' \ '--cwd[Set a specific cwd]:cwd' \ @@ -693,6 +693,7 @@ _bun_dedupe_completion() { _arguments -s -C \ '1: :->cmd1' \ '--check[Exit with code 1 if the lockfile has duplicate versions that can be removed, without changing anything]' \ + '--why[Also list each version'"'"'s dependents and the ranges they asked for]' \ '-c[Load config(bunfig.toml)]: :->config' \ '--config[Load config(bunfig.toml)]: :->config' \ '-y[Write a yarn.lock file (yarn v1)]' \ @@ -713,7 +714,7 @@ _bun_dedupe_completion() { '--no-progress[Disable the progress bar]' \ '--no-summary[Don'"'"'t print a summary]' \ '--no-verify[Skip verifying integrity of newly downloaded packages]' \ - '--ignore-scripts[Skip lifecycle scripts in the package.json (dependency scripts are never run)]' \ + '--ignore-scripts[Skip lifecycle scripts for all packages, including the project'"'"'s package.json and trusted dependencies]' \ '--cwd[Set a specific cwd]:cwd' \ '--backend[Platform-specific optimizations for installing dependencies]:backend:("copyfile" "hardlink" "symlink")' \ '--linker[Linker strategy]:linker:(isolated hoisted)' \ @@ -737,8 +738,10 @@ _bun_prune_completion() { '--prod[Also remove packages that are only needed by devDependencies]' \ '*--omit[Also remove packages that are only needed by the given dependency types]:type:(dev optional peer)' \ '--dry-run[Print what would be removed without deleting anything]' \ + '--check[Exit with code 1 if node_modules has packages that can be removed, without deleting anything]' \ '*--os[Prune for a different operating system than the current one]:os' \ '*--cpu[Prune for a different CPU architecture than the current one]:cpu' \ + '*--libc[Prune for a different libc than the current one]:libc:(glibc musl)' \ '--linker[Prune a node_modules installed with the given linker]:linker:(isolated hoisted)' \ '*--filter[Only prune the node_modules folders of the matching workspaces]:workspace pattern' \ '*-F[Only prune the node_modules folders of the matching workspaces]:workspace pattern' \ diff --git a/docs/guides/install/from-npm-install-to-bun-install.mdx b/docs/guides/install/from-npm-install-to-bun-install.mdx index f97fae445048..1fcf44967f03 100644 --- a/docs/guides/install/from-npm-install-to-bun-install.mdx +++ b/docs/guides/install/from-npm-install-to-bun-install.mdx @@ -155,7 +155,7 @@ bun pm ls ``` ```txt -my-pkg node_modules (781) +my-pkg node_modules (781 installed) ├── @types/node@20.16.5 ├── @types/react@18.3.8 ├── @types/react-dom@18.3.0 diff --git a/docs/pm/catalogs.mdx b/docs/pm/catalogs.mdx index e7893ce0eaf9..8b709a0c3856 100644 --- a/docs/pm/catalogs.mdx +++ b/docs/pm/catalogs.mdx @@ -257,7 +257,7 @@ Bun's lockfile tracks catalog versions, so installs are consistent across enviro ```json bun.lock(excerpt) icon="file-json" { - "lockfileVersion": 2, + "lockfileVersion": 1, "workspaces": { "": { "name": "react-monorepo", diff --git a/docs/pm/cli/audit.mdx b/docs/pm/cli/audit.mdx index 8fa35e1734fb..310cd6d67a08 100644 --- a/docs/pm/cli/audit.mdx +++ b/docs/pm/cli/audit.mdx @@ -72,13 +72,15 @@ The JSON is unfiltered — `--audit-level` and `--ignore` only affect the exit c bun audit fix ``` -Runs the audit, then upgrades each vulnerable package to the lowest non-vulnerable version that every dependent's range allows, and installs. Only `bun.lock` and `node_modules` change, with one exception: Bun treats a direct dependency pinned to an exact version as `^version`. If Bun finds a fix, it rewrites the pin in `package.json` (or the catalog entry). +Runs the audit, then upgrades each vulnerable package to the lowest non-vulnerable version that every dependent's range allows, and installs. Only `bun.lock` and `node_modules` change, with one exception: Bun treats an exact version pin in your own `package.json` (a dependency, a catalog entry, or an [`overrides`](/pm/overrides) rule) as `^version`. If Bun finds a fix, it rewrites the pin. ``` fixing: ms@0.7.0 → 0.7.1 lodash@4.17.20 → 4.17.21 package.json: 4.17.20 → 4.17.21 + tar@6.1.11 → 6.1.12 + package.json (overrides): 6.1.11 → 6.1.12 blocked by a dependent's range: minimatch@0.3.0 → 3.0.2 @@ -86,18 +88,21 @@ blocked by a dependent's range: semver@5.7.1 → 6.3.1 my-app depends on semver@^5.0.0 bun audit fix --latest + qs@6.5.2 → 6.11.0 + package.json overrides qs@~6.5.0 (express>qs) + bun audit fix --latest no published version fixes: left-pad@1.3.0 GHSA-xxxx-xxxx-xxxx bun audit fix --ignore GHSA-xxxx-xxxx-xxxx -Fixed 2 vulnerabilities in 2 packages +Fixed 3 vulnerabilities in 3 packages 5 vulnerabilities remaining ``` -- **blocked by a dependent's range** — no safe version fits a dependent's declared range. If the range is in your own `package.json` or catalog, `bun audit fix --latest` gets past it. Otherwise, update the dependent or add an [`overrides`](/pm/overrides) entry. -- **no published version fixes** — every published version is vulnerable. Replace the package, or silence the advisory with the printed `--ignore` command. -- If no newer version is safe but an older one is, Bun downgrades and marks the row `(downgrade)`. +- **blocked by a dependent's range** — no safe version fits a dependent's declared range. If the range is your own (a `package.json` dependency, a catalog entry, or an `overrides` rule, reported as `package.json overrides ...`), `bun audit fix --latest` gets past it. Otherwise, update the dependent or add an [`overrides`](/pm/overrides) entry. +- **no published version fixes** — every version Bun would install is vulnerable. Replace the package, or silence the advisory with the printed `--ignore` command. +- If no newer version is safe but an older one is, Bun downgrades and marks the row `(downgrade)`. Bun only downgrades within the installed major version (within the installed minor version below 1.0.0), with or without `--latest`. When an advisory covers every `2.x` release, Bun lists the package under **no published version fixes** instead of installing a `1.x` release. - Bun still installs a safe version newer than `--minimum-release-age` and marks the row `(newer than --minimum-release-age)`. - Bun upgrades patched dependencies (`patchedDependencies`) like any other package. Re-create the patch afterwards with `bun patch`. - After installing, Bun re-audits the new lockfile. The `remaining` count and exit code reflect that second audit, so they match what a follow-up `bun audit` would report. @@ -112,10 +117,10 @@ Fixed 2 vulnerabilities in 2 packages bun audit fix --latest ``` -Same as `bun audit fix`, but ranges in your own `package.json` files and catalogs no longer block a fix. Bun rewrites them to accept the new version, keeping their style (`^5.0.0` → `^6.3.1`, `~5.7.1` → `~6.3.1`, exact stays exact). Ranges declared by third-party packages still block; use `overrides` for those. +Same as `bun audit fix`, but ranges in your own `package.json` files, catalogs, and `overrides` no longer block a fix. Bun rewrites them to accept the new version, keeping their style (`^5.0.0` → `^6.3.1`, `~5.7.1` → `~6.3.1`, exact stays exact). Ranges declared by third-party packages still block; use `overrides` for those. ### Exit code `0` if no vulnerabilities remain after Bun applies `--audit-level` and `--ignore`, `1` otherwise. For `bun audit fix`, this is based on the re-audit after installing (or the plan, with `--dry-run`). -If the registry request fails, both commands print `audit request failed` to stderr and exit `1`. +If the registry request fails, both commands print `audit request failed` to stderr and exit `1`. The same goes for a response that is not an advisory report (for example, a registry or proxy that answers the audit request with an HTML page or an error object): a response Bun cannot read never counts as a clean audit. diff --git a/docs/pm/cli/dedupe.mdx b/docs/pm/cli/dedupe.mdx index 67c09efadb3a..5501daa389cc 100644 --- a/docs/pm/cli/dedupe.mdx +++ b/docs/pm/cli/dedupe.mdx @@ -15,7 +15,7 @@ bun dedupe v1.4.0 (abc12345) ↳ esbuild 0.15.10 → 0.15.11 ↳ react 18.2.0 → 18.3.1 -2 duplicate versions removed, 3 packages installed (checked 5 packages) [12.00ms] +2 duplicate versions removed, 3 packages installed (checked 5 packages in bun.lock) [12.00ms] ``` Each row is a version Bun removed and the version its dependents now use. @@ -36,7 +36,7 @@ bun dedupe v1.4.0 (abc12345) ↳ esbuild 0.15.10 → 0.15.11 ↳ react 18.2.0 → 18.3.1 -2 duplicate versions can be removed (checked 5 packages) [9.00ms] +2 duplicate versions can be removed (checked 5 packages in bun.lock) [9.00ms] bun dedupe ``` @@ -44,6 +44,27 @@ bun dedupe v1.4.0 (abc12345) `--lockfile-only` rewrites `bun.lock` without installing. +### `--why` + +`--why` adds one level of dependents under each row: the packages whose dependency edges pointed at the removed version or at the version(s) it moved to, grouped by the range they asked for. + +```bash terminal icon="terminal" +bun dedupe --dry-run --why +``` + +``` +bun dedupe v1.4.0 (abc12345) + +↳ esbuild 0.15.10 → 0.15.11 + wanted ^0.15.7 by esbuild-loader, tsup, vite +1 more + wanted ^0.15.8 by my-app + +1 duplicate version can be removed (checked 5 packages) [9.00ms] + bun dedupe +``` + +This shows at a glance why a version moved where it did: here `my-app`'s `^0.15.8` keeps `0.15.11` alive, and the `^0.15.7` edges are satisfied by it too. When a row has no arrow (the version is dropped because nothing reachable needs it anymore), its listed dependents are themselves removed versions, written as `name@version`. Use [`bun why`](/pm/cli/why) for the full dependency paths of a single package. + ### Notes - Bun respects overrides and catalogs. It re-points each dependency using its effective range. diff --git a/docs/pm/cli/install.mdx b/docs/pm/cli/install.mdx index bdf55958b896..52996dad8474 100644 --- a/docs/pm/cli/install.mdx +++ b/docs/pm/cli/install.mdx @@ -174,6 +174,8 @@ bun install --frozen-lockfile Bun does not enable `--frozen-lockfile` automatically in CI; pass the flag or use `bun ci`. If there is no lockfile at all, `--frozen-lockfile` installs from `package.json` without writing one. +If the project has a `package-lock.json`, `yarn.lock`, or `pnpm-lock.yaml` but no `bun.lock`, `--frozen-lockfile` migrates that lockfile in memory and installs from it. The install fails if the migrated lockfile does not match `package.json`. Bun writes nothing in this case: no `bun.lock`, and none of the `package.json` edits a [pnpm migration](#pnpm-migration) makes. It prints a `note:` instead. To finish the migration, run `bun install` without the flag and commit the files it writes. The `bun.lockb` to `bun.lock` conversion described in [lockfile](/pm/lockfile) is the one write `--frozen-lockfile` still performs. + `--frozen-lockfile` works on a pruned monorepo checkout (e.g. `turbo prune` output, or a Docker context with only some workspace folders copied in). If a workspace listed in `bun.lock` is missing its `package.json` on disk, Bun skips it with a `note:` and does not install its exclusive dependencies. If a remaining workspace depends on a skipped one, the install fails. To validate the lockfile without installing, use `bun install --frozen-lockfile --dry-run`. @@ -206,6 +208,12 @@ To perform a dry run, without installing anything: bun install --dry-run ``` +Bun resolves the dependency tree and prints the summary. During a dry run, Bun does not: + +- install packages into `node_modules` +- write `bun.lock` (or `yarn.lock` with `--yarn`) +- run your project's lifecycle scripts + --- ## Non-npm dependencies @@ -277,10 +285,15 @@ You can also configure this in `bunfig.toml`: # Only install package versions published at least 3 days ago minimumReleaseAge = 259200 # seconds -# Exclude trusted packages from the age gate -minimumReleaseAgeExcludes = ["@types/node", "typescript"] +# Exclude trusted packages, or individual versions you have vetted, from the age gate +minimumReleaseAgeExcludes = ["@types/node", "typescript@5.4.5", "esbuild@0.21.4 || 0.21.5"] ``` +`minimumReleaseAgeExcludes` accepts two kinds of entries: + +- A package name (`"typescript"`) exempts every version of that package, including versions published in the future. +- A package name followed by one or more exact versions (`"typescript@5.4.5"`, `"@types/node@20.11.0"`, `"esbuild@0.21.4 || 0.21.5"`) exempts only those versions. Other versions of the package still go through the age gate, so you can install a specific release early without disabling the gate for the package. Exempted versions are installed as-is: they also skip the stability check described below. Entries whose versions are not exact (`"typescript@^5"`, `"typescript@latest"`, `"typescript@"`) are ignored with a warning. + When the minimum age filter is active: - It only affects new package resolution; existing packages in `bun.lock` remain unchanged @@ -420,22 +433,28 @@ jobs: ## Platform-specific dependencies? -Bun stores normalized `cpu` and `os` values from npm in the lockfile, along with the resolved packages. It skips downloading, extracting, and installing packages disabled for the current target at runtime. This means the lockfile doesn't change between platforms/architectures even if the packages ultimately installed do change. +Bun stores normalized `cpu`, `os` and `libc` values from npm in the lockfile, along with the resolved packages. It skips downloading, extracting, and installing packages disabled for the current target at runtime. This means the lockfile doesn't change between platforms/architectures even if the packages ultimately installed do change. -### `--cpu` and `--os` flags +The `libc` field (`"glibc"` or `"musl"`) is how native packages such as `@rollup/rollup-linux-x64-gnu` and `@rollup/rollup-linux-x64-musl` tell the two Linux variants apart. npm only includes it in the full package metadata, so Bun requests the full metadata for `optionalDependencies`, where these variants are declared, and keeps using the smaller abbreviated metadata for everything else. Accordingly, `libc` is only checked for `optionalDependencies`: a package listed in `dependencies` or `devDependencies` is installed regardless of its `libc` field. By default, Bun on Linux installs the variant for the libc it was built against (the regular build is glibc, the `-musl` build is musl), and Bun on other operating systems does not filter by `libc` at all. The `--libc` flag below overrides this either way, for example to install for a Linux target from macOS. + +### `--cpu`, `--os` and `--libc` flags You can override the target platform for package selection: ```bash -bun install --cpu=x64 --os=linux +bun install --cpu=x64 --os=linux --libc=musl ``` -These flags install packages for the specified platform instead of the current system. Use them for cross-platform builds or when preparing deployments for different environments. +These flags install packages for the specified platform instead of the current system. Use them for cross-platform builds or when preparing deployments for different environments, for example installing on a glibc machine for an Alpine image. **Accepted values for `--cpu`**: `arm`, `arm64`, `ia32`, `mips`, `mipsel`, `ppc`, `ppc64`, `s390`, `s390x`, `x32`, `x64` **Accepted values for `--os`**: `aix`, `darwin`, `freebsd`, `linux`, `openbsd`, `sunos`, `win32`, `android` +**Accepted values for `--libc`**: `glibc`, `musl` + +Each flag can be repeated, takes `*` to allow every value, and takes `!name` to exclude one: `--libc='*'` installs both the glibc and the musl variant. + ## Peer dependencies? Bun handles peer dependencies like Yarn: `bun install` installs them automatically. If the dependency is marked optional in `peerDependenciesMeta`, Bun uses an existing dependency if possible. @@ -516,6 +535,8 @@ bun install Migration only runs when `bun.lock` is absent. There is currently no opt-out flag for pnpm migration. +Bun writes the migrated `bun.lock` and the `package.json` edits described below together, and only when the command saves a lockfile. `bun install`, `bun add`, `bun remove`, `bun update`, `bun pm migrate`, and `bun pm trust` write both files. `bun install --frozen-lockfile` (and `bun ci`), `bun install --dry-run`, `bun install --no-save`, and commands that only read the lockfile, such as `bun outdated` and `bun pm why`, migrate in memory and leave both files untouched. + The migration process handles: ### Lockfile Migration @@ -523,6 +544,8 @@ The migration process handles: - Converts `pnpm-lock.yaml` (lockfile versions 7–9, including pnpm 11's multi-document files) to `bun.lock` - Preserves resolved versions and integrity hashes - Preserves peer dependency ranges and `peerDependenciesMeta`, so the next `bun install` leaves the migrated lockfile unchanged +- Writes every catalog entry declared in `pnpm-workspace.yaml` to `bun.lock`, including entries no workspace uses yet (`pnpm-lock.yaml` lists only the entries in use), so the next `bun install` leaves the catalog unchanged +- Keeps `catalog:` and `workspace:` specifiers as written and records each workspace package's version, so `bun install --frozen-lockfile` accepts the migrated lockfile and the versions pnpm locked for each workspace are the ones installed - Migrates git, GitHub, tarball URL, `file:`, and `npm:` alias dependencies, including transitive ones - Resolves pnpm named registries (`name@registry:version`) via `namedRegistries` in `pnpm-workspace.yaml` - Converts injected workspace packages (`dependenciesMeta.*.injected`) to ordinary workspace dependencies @@ -531,7 +554,7 @@ The migration process handles: ### Workspace Configuration -When a `pnpm-workspace.yaml` file exists, Bun migrates workspace settings to your root `package.json`: +When a `pnpm-workspace.yaml` file exists, Bun migrates workspace settings to your root `package.json`. This happens together with the lockfile migration, and also when there is no `pnpm-lock.yaml` to migrate (it was never committed, or it is too old to convert), as long as `bun.lock` does not exist yet and the root `package.json` has no `workspaces` field of its own: ```yaml pnpm-workspace.yaml icon="file-code" packages: @@ -583,11 +606,15 @@ Bun preserves dependencies that use pnpm's `catalog:` protocol: ### Configuration Migration -Bun migrates the following pnpm configuration from both `pnpm-lock.yaml` and `pnpm-workspace.yaml`: +Bun copies the following pnpm configuration into the root `package.json`: + +- **Overrides**: Copied from `pnpm.overrides` to root-level `overrides` in `package.json` +- **Patched Dependencies**: Copied from `pnpm.patchedDependencies` to root-level `patchedDependencies` in `package.json` +- **Workspace Overrides**: Copied from `overrides` and `patchedDependencies` in `pnpm-workspace.yaml` to the same root-level fields + +Bun leaves the `pnpm` field of `package.json` as it is. pnpm reads its configuration from that field and ignores the root-level fields, so `pnpm install --frozen-lockfile` keeps working for teammates who still use pnpm. Bun reads the root-level fields and ignores the `pnpm` field. -- **Overrides**: Moved from `pnpm.overrides` to root-level `overrides` in `package.json` -- **Patched Dependencies**: Moved from `pnpm.patchedDependencies` to root-level `patchedDependencies` in `package.json` -- **Workspace Overrides**: Applied from `pnpm-workspace.yaml` to root `package.json` +- **Allowed build scripts**: `onlyBuiltDependencies` (from `pnpm-workspace.yaml` or `pnpm.onlyBuiltDependencies` in `package.json`) is added to [`trustedDependencies`](/pm/lifecycle) in `package.json`, so the same packages keep running their lifecycle scripts ### Requirements and limitations @@ -596,9 +623,9 @@ Bun migrates the following pnpm configuration from both `pnpm-lock.yaml` and `pn - All catalog entries referenced by dependencies must exist in the catalogs definition - Every workspace in `pnpm-lock.yaml` must have its `package.json` on disk (in Docker, copy them in before `bun install`) - Relative `link:` dependencies and git dependencies with a sub-directory (`resolution.path`) are not supported -- If migration fails for any of these reasons, Bun prints why and resolves from scratch instead +- If migration fails for any of these reasons, Bun prints why and resolves from scratch instead. The `pnpm-workspace.yaml` settings above are still migrated, so workspaces, catalogs and overrides are not lost -After migration, you can safely remove `pnpm-lock.yaml` and `pnpm-workspace.yaml` files. +Once nobody on the repository uses pnpm anymore, you can remove `pnpm-lock.yaml`, `pnpm-workspace.yaml`, and the `pnpm` field of `package.json`. --- diff --git a/docs/pm/cli/link.mdx b/docs/pm/cli/link.mdx index 246b17275992..9f0d5d1040b7 100644 --- a/docs/pm/cli/link.mdx +++ b/docs/pm/cli/link.mdx @@ -31,7 +31,11 @@ cd /path/to/my-app bun link cool-pkg ``` -The `--save` flag also adds `cool-pkg` to the `dependencies` field of your app's package.json, with a version specifier that tells Bun to load from the registered local directory instead of installing from `npm`: +Pass `--save` to also add `cool-pkg` to the `dependencies` field of your app's package.json, with a version specifier that tells Bun to load from the registered local directory instead of installing from `npm`: + +```bash terminal icon="terminal" +bun link --save cool-pkg +``` ```json package.json icon="file-json" { diff --git a/docs/pm/cli/pm.mdx b/docs/pm/cli/pm.mdx index f937d84b6b07..ee0255597ae2 100644 --- a/docs/pm/cli/pm.mdx +++ b/docs/pm/cli/pm.mdx @@ -118,7 +118,7 @@ bun list ``` ```txt -/path/to/project node_modules (135) +/path/to/project node_modules (135 installed) ├── eslint@8.38.0 ├── react@18.2.0 ├── react-dom@18.2.0 @@ -135,7 +135,7 @@ bun list --all ``` ```txt -/path/to/project node_modules (135) +/path/to/project node_modules (135 installed) ├── @eslint-community/eslint-utils@4.4.0 ├── @eslint-community/regexpp@4.5.0 ├── @eslint/eslintrc@2.0.2 @@ -159,7 +159,7 @@ bun list --trusted ``` ```txt -/path/to/project node_modules (135) +/path/to/project node_modules (135 installed) └── esbuild@0.21.5 ``` @@ -174,6 +174,8 @@ bun pm licenses ls ``` ```txt +bun pm licenses v1.3.0 (a4b2f86f) + MIT (2) ├── path-parse@1.0.6 └── resolve@1.9.0 @@ -183,6 +185,8 @@ Unknown (4) ├── no-deps@1.0.0 ├── no-deps@1.0.1 └── one-dep@1.0.0 + +6 packages across 2 licenses (checked 6 packages in bun.lock) [4.00ms] ``` | Flag | Description | @@ -379,6 +383,8 @@ bun pm pkg get scripts.build # nested property bun pm pkg set name="my-package" # simple property bun pm pkg set scripts.test="jest" version=2.0.0 # multiple properties bun pm pkg set private=true --json # JSON values with --json flag +bun pm pkg set 'keywords[]=cli' # append to an array (created if missing) +bun pm pkg set 'contributors[0].name=Jane' # update an array element # delete bun pm pkg delete description # single property diff --git a/docs/pm/cli/prune.mdx b/docs/pm/cli/prune.mdx index 3988e1a0b28a..40f2674b02c9 100644 --- a/docs/pm/cli/prune.mdx +++ b/docs/pm/cli/prune.mdx @@ -16,7 +16,7 @@ bun prune v1.4.0 (abc12345) - @types/node@20.11.5 - left-pad@1.3.0 -2 packages removed (checked 948) [22.00ms] +2 packages removed (checked 948 installed packages) [22.00ms] ``` Packages removed from a workspace or nested `node_modules` folder show the folder in parentheses, e.g. `- typescript@5.4.0 (packages/app/node_modules)`. @@ -47,10 +47,18 @@ bun prune --production --dry-run bun prune v1.4.0 (abc12345) - typescript@5.4.0 -1 package can be removed (checked 948) [9.00ms] +1 package can be removed (checked 948 installed packages) [9.00ms] bun prune --production ``` +### `--check` + +Print the same report as `--dry-run`, but exit `1` if anything would be removed. Use it in CI: + +```bash terminal icon="terminal" +bun prune --production --check +``` + ### `--filter` Prune only the selected workspaces' `node_modules` folders (same patterns as [`bun install --filter`](/pm/filter)). Bun also cleans shared locations: the root `node_modules`, or `node_modules/.bun` with the isolated linker. In those locations, Bun keeps anything an unselected workspace still needs. @@ -66,7 +74,7 @@ bun prune --production --filter app - Uses the same linker as `bun install` would. If `node_modules` was created with the other linker, `bun prune` refuses to run. Pass the matching `--linker`, or run `bun install`. - Matches packages by name. If a package is at the wrong version, Bun leaves it for `bun install` to replace. Bun only removes a nested copy (`node_modules/a/node_modules/b`) once the correct version is installed above it; otherwise Bun keeps it and prints a warning. - Never removes workspace folders, `.bin` entries still in use, dot-directories like `.cache`, plain files, or anything outside `node_modules`. -- Removes packages disabled for the current `os`/`cpu`. Pass `--os`/`--cpu` to prune for another platform. +- Removes packages disabled for the current `os`/`cpu`/`libc`. Pass `--os`/`--cpu`/`--libc` to prune for another platform. - Works on a pruned monorepo checkout (e.g. `turbo prune` output) the same way `bun install --frozen-lockfile` does. - If any entry fails to delete, the command still removes the rest and exits `1`. - `--global` is not supported. diff --git a/docs/pm/cli/publish.mdx b/docs/pm/cli/publish.mdx index a27e89a2cf0f..2139888a1f03 100644 --- a/docs/pm/cli/publish.mdx +++ b/docs/pm/cli/publish.mdx @@ -5,7 +5,7 @@ description: Use `bun publish` to publish a package to the npm registry import Publish from "/snippets/cli/publish.mdx"; -`bun publish` packs your package into a tarball and strips catalog and workspace protocols from the `package.json`, resolving versions if necessary. It then publishes to the registry specified in your configuration files. Both `bunfig.toml` and `.npmrc` files are supported. +`bun publish` packs your package into a tarball and strips catalog and workspace protocols from the `package.json`, resolving versions if necessary. It then publishes to the registry specified in your configuration files or in the package's `publishConfig`. Both `bunfig.toml` and `.npmrc` files are supported. ```sh terminal icon="terminal" ## Publishing the package from the current working directory @@ -81,6 +81,32 @@ You can also set `--tag` in the `publishConfig` field of your `package.json`. } ``` +### `publishConfig.registry` + +A package can pin the registry it is published to in the `publishConfig` field of its `package.json`, regardless of the registry configured for installs. `--registry` on the command line still overrides it. + +```json package.json icon="file-json" +{ + "name": "internal-package", + "publishConfig": { + "registry": "https://registry.myorg.example/" + } +} +``` + +As in npm, `publishConfig.registry` replaces the default registry only. A registry configured for the package's scope (`@myorg:registry` in `.npmrc`, or `[install.scopes]` in `bunfig.toml`) still takes precedence over it; to replace that one, set the scope's key in `publishConfig` instead: + +```json package.json icon="file-json" +{ + "name": "@myorg/package", + "publishConfig": { + "@myorg:registry": "https://registry.myorg.example/" + } +} +``` + +Credentials are the ones configured for that registry, the same as when it is the default registry: a `//registry.myorg.example/:_authToken=...` line in `.npmrc`, or a registry with the same URL in `bunfig.toml` or `.npmrc`. When there are none, the credentials of the registry the package would otherwise have been published to are used only if the pinned registry has the same origin; otherwise `bun publish` fails with a missing authentication error instead of sending them to a different registry. + ### `--dry-run` `--dry-run` runs the publish process without publishing the package, so you can verify what would be published. diff --git a/docs/pm/cli/update.mdx b/docs/pm/cli/update.mdx index 169b8b4e8d6d..580dd0eab3a7 100644 --- a/docs/pm/cli/update.mdx +++ b/docs/pm/cli/update.mdx @@ -36,6 +36,7 @@ Updated packages appear in the install summary as `↑ name old → new`, with ` ### What is held back - Bun never widens ranges. A package that depends on `foo@^1.0.0` never gets `foo@2.x`. +- A transitive dependency that shares the copy your own `package.json` entry resolves to (in the root or in a workspace) stays on that copy and moves with it. With `@types/node: ~20` in your `package.json`, the `@types/node: *` that `@types/ws` declares keeps using your `~20` copy instead of nesting the newest major under `@types/ws`. Bun updates a transitive range on its own when it rejects the version your entry moves to, or when you removed the entry. - Versions in `patchedDependencies` stay put as long as their range allows. Bun reports them as `kept name@version (patched, v1.2.3 available)`. `--latest` and [`bun audit fix`](/pm/cli/audit#bun-audit-fix) do move them; re-create the patch with [`bun patch`](/pm/cli/patch) afterwards. - If a registry request for a transitive package fails, that package keeps its locked version and Bun prints a warning. A failed request for a direct dependency is an error. diff --git a/docs/pm/cli/why.mdx b/docs/pm/cli/why.mdx index 2091605c8c99..6db73b4723f4 100644 --- a/docs/pm/cli/why.mdx +++ b/docs/pm/cli/why.mdx @@ -82,3 +82,21 @@ The output shows: - The version requirement specified in each package's dependencies For nested dependencies, the command shows the complete dependency tree by default, with indentation indicating the relationship hierarchy. + +A package can appear in the tree more than once when several chains go through it. Once its dependents have been listed (and take up more than one line), later occurrences are marked `*deduped` instead of listing them again. The exception is `--depth`: if the listing was cut off by the depth limit and the package appears again closer to the queried package, where more levels fit, it is listed again. A dependency cycle is marked `*circular`. + +```bash terminal icon="terminal" +bun why pkg-x +``` + +```txt +pkg-x@workspace:packages/pkg-x + ├─ monorepo + ├─ pkg-a@workspace (requires workspace:*) + │ └─ pkg-c@workspace (requires workspace:*) + │ └─ pkg-d@workspace (requires workspace:*) + │ └─ pkg-f@workspace (requires workspace:*) + │ + └─ pkg-c@workspace (requires workspace:*) + └─ *deduped +``` diff --git a/docs/pm/isolated-installs.mdx b/docs/pm/isolated-installs.mdx index 0f322aa295ba..669a0b5dd25c 100644 --- a/docs/pm/isolated-installs.mdx +++ b/docs/pm/isolated-installs.mdx @@ -184,6 +184,8 @@ If you encounter issues, you can: bun install --linker hoisted ``` + Switching linkers in either direction reinstalls the existing `node_modules` from scratch. The previous tree is moved to `node_modules/.old_modules-`, which is safe to delete. + 2. **Report compatibility issues** to help improve isolated install support ### Performance considerations @@ -209,13 +211,15 @@ bun install --linker isolated Isolated installs are conceptually similar to pnpm, so migration is direct: ```bash terminal icon="terminal" -# Remove pnpm files -rm -rf node_modules pnpm-lock.yaml +# Remove pnpm's node_modules +rm -rf node_modules # Install with Bun's isolated linker bun install --linker isolated ``` +Keep `pnpm-lock.yaml` around for this first install: Bun converts it to `bun.lock`, so you keep the versions pnpm had resolved. The same first install (no `bun.lock` yet, no `workspaces` field in the root `package.json`) also moves the workspace list, catalogs and overrides from `pnpm-workspace.yaml` into the root `package.json`, with or without a `pnpm-lock.yaml`. See [pnpm migration](/pm/cli/install#pnpm-migration) for what is converted. + The layouts are close: Bun hardlinks (clones on macOS) packages from its [global cache](/pm/global-cache) into a per-project store, `node_modules/.bun/`, and symlinks top-level `node_modules` entries into it. With [`install.globalStore`](#global-virtual-store) enabled, those store entries become symlinks into a global virtual store instead. ## When to use isolated installs diff --git a/docs/pm/lockfile.mdx b/docs/pm/lockfile.mdx index 5bec794a3ad5..e8e6beb40883 100644 --- a/docs/pm/lockfile.mdx +++ b/docs/pm/lockfile.mdx @@ -11,7 +11,7 @@ Yes #### Generate a lockfile without installing? -To generate a lockfile without installing to `node_modules`, use the `--lockfile-only` flag. Bun always saves the lockfile to disk, even if it is already up to date with your project's `package.json`(s). The exception is when `--frozen-lockfile` (or `--production`) is set. +To generate a lockfile without installing to `node_modules`, use the `--lockfile-only` flag. Bun always saves the lockfile to disk, even if it is already up to date with your project's `package.json`(s). The exceptions are `--frozen-lockfile` (or `--production`), `--dry-run`, and `--no-save`, which save nothing. ```bash terminal icon="terminal" bun install --lockfile-only @@ -54,14 +54,22 @@ Bun v1.2 changed the default lockfile format to the text-based `bun.lock`. To mi For more on the format, see [the blog post](https://bun.com/blog/bun-lock-text-lockfile). +#### `lockfileVersion` + +The `lockfileVersion` field at the top of `bun.lock` records its format version. A new `bun.lock` is written as `lockfileVersion` 1, which every Bun release since v1.2 can read, and re-saving a `lockfileVersion` 1 or 2 lockfile keeps its version. The exception is [nested or version-scoped overrides](/pm/overrides#limitations): while a lockfile contains those it is written as `lockfileVersion` 3, which requires Bun v1.4 or later, and it goes back to 1 once they are removed. + #### Automatic lockfile migration When you run `bun install` in a project without a `bun.lock`, Bun automatically migrates existing lockfiles: - `yarn.lock` (v1) -- `package-lock.json` (npm, `lockfileVersion` 2, 3 or 4) +- `npm-shrinkwrap.json` and `package-lock.json` (npm, `lockfileVersion` 2, 3 or 4) - `pnpm-lock.yaml` (pnpm) -Bun does not migrate a `package-lock.json` from npm 6 or older (`lockfileVersion` 1); it prints a warning and resolves from `package.json` instead. +When a project has both npm lockfiles, Bun reads `npm-shrinkwrap.json` and ignores `package-lock.json`, as npm does. + +Bun does not migrate an npm lockfile from npm 6 or older (`lockfileVersion` 1); it prints a warning and resolves from `package.json` instead. Bun preserves the original lockfile. You can remove it manually after verification. + +Bun writes the migrated `bun.lock` only when the command saves a lockfile. `bun install --frozen-lockfile` (and `bun ci`), `--dry-run`, `--no-save`, and read-only commands such as `bun outdated` use the migrated lockfile in memory and write nothing. Under `--frozen-lockfile`, Bun prints a `note:` asking you to run `bun install` and commit the result. To migrate without installing, run `bun pm migrate` or `bun install --lockfile-only`. diff --git a/docs/pm/npmrc.mdx b/docs/pm/npmrc.mdx index 6e734c9f32bc..0057cc54a586 100644 --- a/docs/pm/npmrc.mdx +++ b/docs/pm/npmrc.mdx @@ -13,7 +13,7 @@ Configuration is loaded in this order, with later sources overriding earlier one 4. `BUN_CONFIG_REGISTRY` / `NPM_CONFIG_REGISTRY` and `BUN_CONFIG_TOKEN` / `NPM_CONFIG_TOKEN` environment variables 5. Command-line flags such as `--registry` -Bun matches credentials in `.npmrc` (`///:_authToken`, etc.) to registries by host and path, even if you set the registry URL itself in `bunfig.toml`. +Bun matches credentials in `.npmrc` (`///:_authToken`, etc.) by host and path to the URLs it requests, so they apply no matter where you set the registry URL itself (`.npmrc`, `bunfig.toml`, an environment variable or `--registry`). Bun also uses them for tarballs a registry serves from a different host than its own, as long as that host has its own `///:_authToken` line; it never sends a registry's credentials to another host. Values may reference environment variables. Bun replaces `${NAME}` with the variable's value, or leaves it as-is if the variable is unset. `${NAME?}` becomes an empty string if unset. @@ -24,6 +24,21 @@ Values may reference environment variables. Bun replaces `${NAME}` with the vari --- +## Which files are read + +Bun reads up to two `.npmrc` files. Options in the later file override the earlier one: + +1. The user-level file. This is the file `NPM_CONFIG_USERCONFIG` (npm's `userconfig` option) points at when that variable is set; otherwise `$XDG_CONFIG_HOME/.npmrc` if it exists, otherwise `$HOME/.npmrc`. +2. The `.npmrc` next to your project's root `package.json`. + + + `actions/setup-node` with `registry-url` writes an `.npmrc` to `$RUNNER_TEMP` and exports `NPM_CONFIG_USERCONFIG` + pointing at it, so `bun publish` reads the token from `NODE_AUTH_TOKEN` in that workflow the same way `npm publish` + does. + + +--- + ## Supported options ### Set the default registry diff --git a/docs/pm/overrides.mdx b/docs/pm/overrides.mdx index ee4357e3ac24..40f5b7b2a335 100644 --- a/docs/pm/overrides.mdx +++ b/docs/pm/overrides.mdx @@ -62,7 +62,7 @@ Bun only reads overrides from the root `package.json`, not from workspace packag ## `"resolutions"` -`"resolutions"` is Yarn's alternative to `"overrides"`, with similar syntax. Bun supports it to help projects migrate from Yarn. +`"resolutions"` is Yarn's alternative to `"overrides"`, with similar syntax. Bun supports it to help projects migrate from Yarn. When a `package.json` has both fields, Bun applies the rules from both. If the same package (or the same nested selector) appears in both, the `"overrides"` rule wins. {/* prettier-ignore */} ```json package.json icon="file-json" diff --git a/docs/pm/workspaces.mdx b/docs/pm/workspaces.mdx index 7673b6111780..da2d68697713 100644 --- a/docs/pm/workspaces.mdx +++ b/docs/pm/workspaces.mdx @@ -66,6 +66,8 @@ Each workspace has its own `package.json`. To reference another package in the m } ``` +A workspace needs a `"name"`. Bun skips a matched `package.json` that has no `"name"`, such as a test fixture containing only `{ "type": "module" }`. Bun does not link the skipped directory into `node_modules` and does not install its dependencies. If the skipped `package.json` declares dependencies, `bun install` prints a warning that names the directory. + `bun install` installs dependencies for all workspaces in the monorepo, de-duplicating packages if possible. To install dependencies for specific workspaces only, use the `--filter` flag. ```bash @@ -90,6 +92,21 @@ A specific version takes precedence over the package's `package.json` version: "workspace:1.0.2" -> "1.0.2" // Even if current version is 1.0.1 ``` +You can also reference a workspace package by the path of its directory. The path is relative to the `package.json` that declares the dependency. Bun publishes the version of the package in that directory. When the dependency name differs from the package name, Bun publishes an `npm:` alias: + +``` +"pkg-b": "workspace:../pkg-b" -> "pkg-b": "1.0.1" +"b": "workspace:../pkg-b" -> "b": "npm:pkg-b@1.0.1" +``` + +To install a workspace package under a different name, put the package name in front of the version: `workspace:@`. Bun publishes these as `npm:` aliases too: + +``` +"b": "workspace:pkg-b@*" -> "b": "npm:pkg-b@1.0.1" +"b": "workspace:pkg-b@^" -> "b": "npm:pkg-b@^1.0.1" +"b": "workspace:pkg-b@1.0.1" -> "b": "npm:pkg-b@1.0.1" +``` + Workspaces have a few major benefits. - **Split code into logical parts.** If one package relies on another, add it as a dependency in `package.json`. If package `b` depends on `a`, `bun install` installs your local `packages/a` directory into `node_modules` instead of downloading it from the npm registry. diff --git a/docs/runtime/bunfig.mdx b/docs/runtime/bunfig.mdx index 5927b01bf6d6..f6f385cdae4b 100644 --- a/docs/runtime/bunfig.mdx +++ b/docs/runtime/bunfig.mdx @@ -756,15 +756,17 @@ See [Minimum release age](/pm/cli/install#minimum-release-age). ### `install.minimumReleaseAgeExcludes` -An array of package names that are exempt from the `minimumReleaseAge` check. Default `[]`. +An array of packages that are exempt from the `minimumReleaseAge` check. A package name exempts every version of the package; `name@version` (or `name@1.0.0 || 1.0.1`) exempts only the listed exact versions. Default `[]`. ```toml title="bunfig.toml" icon="settings" [install] minimumReleaseAge = 259200 -# These packages will bypass the 3-day minimum age requirement -minimumReleaseAgeExcludes = ["@types/bun", "typescript"] +# Every version of @types/bun, and only typescript 5.4.5, bypass the 3-day minimum age requirement +minimumReleaseAgeExcludes = ["@types/bun", "typescript@5.4.5"] ``` +See [Minimum release age](/pm/cli/install#minimum-release-age). + ## `bun run` The `[run]` section configures the `bun run` command. These settings also apply to the `bun` command when running a file, script, or executable. diff --git a/docs/runtime/environment-variables.mdx b/docs/runtime/environment-variables.mdx index aaf6d5beda5f..40b0a3545d67 100644 --- a/docs/runtime/environment-variables.mdx +++ b/docs/runtime/environment-variables.mdx @@ -203,7 +203,7 @@ Bun reads these environment variables to configure aspects of its behavior. | `TMPDIR` | Bun occasionally requires a directory to store intermediate assets during bundling or other operations. If unset, defaults to the platform-specific temporary directory: `/tmp` on Linux, `/private/tmp` on macOS. | | `NO_COLOR` | If `NO_COLOR=1`, then ANSI color output is [disabled](https://no-color.org/). | | `FORCE_COLOR` | If `FORCE_COLOR=1`, then ANSI color output is forced on, even if `NO_COLOR` is set. | -| `BUN_CONFIG_MAX_HTTP_REQUESTS` | Sets the maximum number of concurrent HTTP requests sent by fetch and `bun install`. Defaults to `256`. Lower it if you run into rate limits or connection issues. | +| `BUN_CONFIG_MAX_HTTP_REQUESTS` | Sets the maximum number of concurrent HTTP requests sent by `fetch` and `bun install`. `fetch` defaults to `256`. `bun install` defaults to `64`, and its `--network-concurrency` flag overrides this variable. Lower it if you run into rate limits or connection issues. | | `BUN_CONFIG_NO_CLEAR_TERMINAL_ON_RELOAD` | If `BUN_CONFIG_NO_CLEAR_TERMINAL_ON_RELOAD=true`, then `bun --watch` does not clear the console on reload | | `DO_NOT_TRACK` | Disable uploading crash reports to `bun.report` on crash. On macOS & Windows, crash report uploads are enabled by default. Bun sends no other telemetry, though we plan to add some. If `DO_NOT_TRACK=1`, then auto-uploading crash reports and telemetry are both [disabled](https://do-not-track.dev/). | | `BUN_OPTIONS` | Prepends command-line arguments to any Bun execution. For example, `BUN_OPTIONS="--hot"` makes `bun run dev` behave like `bun --hot run dev`. | diff --git a/docs/snippets/cli/add.mdx b/docs/snippets/cli/add.mdx index 170d397c69e1..5a2e18bc855e 100644 --- a/docs/snippets/cli/add.mdx +++ b/docs/snippets/cli/add.mdx @@ -116,7 +116,7 @@ bun add <@version> variables - + Maximum number of concurrent network requests diff --git a/docs/snippets/cli/install.mdx b/docs/snippets/cli/install.mdx index c8392f9f27ff..f65e5026082e 100644 --- a/docs/snippets/cli/install.mdx +++ b/docs/snippets/cli/install.mdx @@ -157,7 +157,7 @@ bun install @ Maximum number of concurrent jobs for lifecycle scripts (default: 2x CPU cores) - + Maximum number of concurrent network requests diff --git a/docs/snippets/cli/link.mdx b/docs/snippets/cli/link.mdx index 6e6d84d3bacb..107c7abf34cb 100644 --- a/docs/snippets/cli/link.mdx +++ b/docs/snippets/cli/link.mdx @@ -39,11 +39,11 @@ bun link - Has no effect. bun link updates package.json and the lockfile only with --save + Don't update package.json or save a lockfile (the default) - Save to package.json + Update package.json and save a lockfile (false by default) @@ -93,7 +93,7 @@ bun link variables - + Maximum number of concurrent network requests @@ -147,6 +147,10 @@ bun link all) + + Override libc for optional dependencies (e.g., glibc, musl, * for all) + + ### Global Configuration & Context diff --git a/docs/snippets/cli/outdated.mdx b/docs/snippets/cli/outdated.mdx index 32d5e582ed78..ceb98222f41b 100644 --- a/docs/snippets/cli/outdated.mdx +++ b/docs/snippets/cli/outdated.mdx @@ -102,8 +102,8 @@ bun outdated Use a specific registry by default, overriding .npmrc, bunfig.toml and environment variables - - Maximum number of concurrent network requests (default 48) + + Maximum number of concurrent network requests (default 64) ### Caching diff --git a/docs/snippets/cli/patch.mdx b/docs/snippets/cli/patch.mdx index 01bf9b04d288..1f74a5270ec0 100644 --- a/docs/snippets/cli/patch.mdx +++ b/docs/snippets/cli/patch.mdx @@ -105,8 +105,8 @@ bun patch @ variables - - Maximum number of concurrent network requests (default 48) + + Maximum number of concurrent network requests (default 64) ### Performance & Resource @@ -159,6 +159,10 @@ bun patch @ all) + + Override libc for optional dependencies (e.g., glibc, musl, * for all) + + ### Global Configuration & Context diff --git a/docs/snippets/cli/publish.mdx b/docs/snippets/cli/publish.mdx index 3989732fa75a..7c3805e00ffb 100644 --- a/docs/snippets/cli/publish.mdx +++ b/docs/snippets/cli/publish.mdx @@ -104,6 +104,16 @@ bun publish --otp 123456 bun publish --registry https://my-private-registry.com ``` +A package can pin the registry it is published to with `publishConfig.registry` in its `package.json` (`--registry` still overrides it). Like npm, this replaces the default registry only; a registry configured for the package's scope takes precedence unless `publishConfig` also sets `@scope:registry`. + +```json package.json icon="file-json" +{ + "publishConfig": { + "registry": "https://my-private-registry.com" // [!code ++] + } +} +``` + #### SSL Certificates @@ -177,7 +187,7 @@ bun publish --cafile ./ca-cert.pem `copyfile` - + Maximum concurrent network requests diff --git a/docs/snippets/cli/remove.mdx b/docs/snippets/cli/remove.mdx index d736f537d175..5dac16332867 100644 --- a/docs/snippets/cli/remove.mdx +++ b/docs/snippets/cli/remove.mdx @@ -146,6 +146,6 @@ bun remove macOS), hardlink (default on Linux and Windows), symlink, copyfile - - Maximum number of concurrent network requests (default 48) + + Maximum number of concurrent network requests (default 64) diff --git a/docs/snippets/cli/update.mdx b/docs/snippets/cli/update.mdx index 913251ada4f8..ae98afaf0b1c 100644 --- a/docs/snippets/cli/update.mdx +++ b/docs/snippets/cli/update.mdx @@ -78,8 +78,8 @@ bun up Use a specific registry by default, overriding .npmrc, bunfig.toml and environment variables - - Maximum number of concurrent network requests (default 48) + + Maximum number of concurrent network requests (default 64) ### Caching diff --git a/mordant-baseline.toml b/mordant-baseline.toml index 81530762db89..7715c528884f 100644 --- a/mordant-baseline.toml +++ b/mordant-baseline.toml @@ -10,7 +10,6 @@ [bun_css] [bun_install] -"defaulted_failure:src/install/npm.rs" = 1 "interchangeable_aliases:src/install/lockfile/Tree.rs" = 1 [bun_js_printer] @@ -35,8 +34,6 @@ "field_valid_only_when:src/runtime/shell/builtin/rm.rs" = 1 "reimplemented_helper:src/runtime/api/bun/Terminal.rs" = 1 "reimplemented_helper:src/runtime/hw_exports.rs" = 1 -"same_match_twice:src/runtime/cli/pack_command.rs" = 1 -"same_match_twice:src/runtime/cli/update_interactive_command.rs" = 2 "same_match_twice:src/runtime/webcore/Blob.rs" = 2 "unchecked_construction:src/runtime/api/js_bundle_completion_task.rs" = 1 "unchecked_construction:src/runtime/cli/pack_command.rs" = 2 diff --git a/scripts/agent.mjs b/scripts/agent.mjs index ca4e7bda731c..4071b6c06a3e 100755 --- a/scripts/agent.mjs +++ b/scripts/agent.mjs @@ -4,6 +4,7 @@ import { copyFileSync, existsSync, readFileSync, realpathSync } from "node:fs"; import { join } from "node:path"; +import { setTimeout as setTimeoutPromise } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; import { @@ -21,11 +22,13 @@ import { getKernel, getOs, homedir, + isLinux, isMacOS, isPosix, isWindows, mkdir, spawnSafe, + spawnSync, which, writeFile, } from "./utils.mjs"; @@ -105,6 +108,15 @@ async function doBuildkiteAgent(action, cliOptions = {}) { } if (isOpenRc()) { + // openrc starts a runlevel's unrelated services one at a time in name + // order, so without `after docker` this service ("b") starts, and the + // ephemeral agent acquires its job, before the boot has reached "docker"; + // the shard then begins with no daemon socket at all (routine on the + // alpine test agents, occasionally for minutes when a service in between + // is slow). `after` is ordering only: images without docker still boot + // the agent. Alpine's docker service returns as soon as dockerd is + // spawned, so start() below additionally waits for the daemon to answer. + // Both take effect when the images are next published. const servicePath = "/etc/init.d/buildkite-agent"; const service = `#!/sbin/openrc-run name="buildkite-agent" @@ -123,6 +135,7 @@ async function doBuildkiteAgent(action, cliOptions = {}) { depend() { need net use dns logger + after docker } `; writeFile(servicePath, service, { mode: 0o755 }); @@ -390,6 +403,19 @@ async function doBuildkiteAgent(action, cliOptions = {}) { .map(([key, value]) => `${key}=${value}`) .join(","); + // The job this machine was booted for starts the moment the agent + // registers (acquire-job above), and the Linux test jobs need dockerd. The + // init system only orders this service after docker's start script (see + // install()), which on alpine returns as soon as dockerd is spawned, so + // give the daemon a bounded head start here: registering later only + // delays the job, registering before the daemon listens starts the job + // against a dead socket (test/docker/index.ts has the history). Past the + // budget the agent registers anyway and the job's "--- Docker" section + // reports the daemon's state. + if (isLinux && which("docker")) { + await waitForDockerDaemon(2 * 60_000); + } + await spawnSafe( [ command, @@ -410,6 +436,29 @@ async function doBuildkiteAgent(action, cliOptions = {}) { } } +/** + * Polls `docker version` (which fails until the daemon is listening) once a + * second for up to `budgetMs`. Returns either way; the outcome is logged. + * @param {number} budgetMs + */ +async function waitForDockerDaemon(budgetMs) { + const startedAt = Date.now(); + const elapsed = () => `${Math.round((Date.now() - startedAt) / 1000)}s`; + const reachable = () => spawnSync(["docker", "version"], { timeout: 10_000 }).exitCode === 0; + if (reachable()) { + return; + } + console.log(`Docker daemon is not reachable yet; waiting up to ${Math.round(budgetMs / 1000)}s for it...`); + while (Date.now() - startedAt < budgetMs) { + await setTimeoutPromise(1_000); + if (reachable()) { + console.log(`Docker daemon became reachable after ${elapsed()}`); + return; + } + } + console.warn(`Docker daemon still unreachable after ${elapsed()}; starting the agent without it`); +} + /** * @returns {boolean} */ diff --git a/scripts/utils.mjs b/scripts/utils.mjs index 713e8ecb6d9e..3594920fbd06 100755 --- a/scripts/utils.mjs +++ b/scripts/utils.mjs @@ -2781,7 +2781,29 @@ export function printEnvironment() { startGroup("Docker", () => { const shell = which(["sh", "bash"]); if (shell) { - spawnSync([shell, "-c", "docker ps"], { stdio: "inherit" }); + const { exitCode } = spawnSync([shell, "-c", "docker ps"], { stdio: "inherit" }); + if (exitCode !== 0) { + // The agent can start the job before the boot has brought dockerd + // up (scripts/agent.mjs orders and waits for it on images published + // after that change; test/docker/coordinator.ts waits for it on + // every image, so this is not fatal). On openrc the service table + // shows which boot step the daemon was still behind. + if (which("rc-status")) { + spawnSync([shell, "-c", "rc-status"], { stdio: "inherit" }); + } + // One line per affected job under a single per-build annotation, so + // how often this still happens can be read off builds (and compared + // across image publishes) without grepping job logs. Info style: + // nothing has failed, and `bun run ci:errors` lists only error-style + // annotations. + reportAnnotationToBuildKite({ + context: "docker-daemon-not-up-at-job-start", + label: "docker daemon not up at job start", + content: `- dockerd was not up when [${getBuildLabel()}](${getBuildUrl()}) started on \`${getHostname()}\`\n`, + style: "info", + priority: 1, + }); + } } }); } diff --git a/src/api/lib.rs b/src/api/lib.rs index 5ea6fdaf5719..1c8639c80dcf 100644 --- a/src/api/lib.rs +++ b/src/api/lib.rs @@ -9,7 +9,7 @@ // ────────────────────────────────────────────────────────────────────────── pub use bun_options_types::schema::api::{ - BunInstall, Ca, NodeLinker, NpmRegistry, NpmRegistryMap, PnpmMatcher, + BunInstall, Ca, NodeLinker, NpmRegistry, NpmRegistryMap, NpmUrlAuth, PnpmMatcher, }; // ────────────────────────────────────────────────────────────────────────── diff --git a/src/bun_core/Progress.rs b/src/bun_core/Progress.rs index a80dbf1ea1b7..44588bf6af5e 100644 --- a/src/bun_core/Progress.rs +++ b/src/bun_core/Progress.rs @@ -194,10 +194,7 @@ pub enum Unit { pub struct Node { pub(crate) context: *mut Progress, pub(crate) parent: *mut Node, - // The non-allocating design means `Node` cannot own the bytes. `'static` - // is the chosen simplification because all current callers (install/, - // cli/) pass string literals; the alternative would be threading a - // lifetime through `Node`/`Progress`. + // Dynamic names (install, create) point at caller-owned scratch buffers that outlive the node. pub name: &'static [u8], pub unit: Unit, /// Must be handled atomically to be thread-safe. diff --git a/src/bun_core/env_var.rs b/src/bun_core/env_var.rs index 1c5082acfb23..3698c291e64c 100644 --- a/src/bun_core/env_var.rs +++ b/src/bun_core/env_var.rs @@ -39,6 +39,10 @@ use crate::ZStr; // ────────────────────────────────────────────────────────────────────────────── new!(pub AGENT: string, "AGENT", {}); +// Set by Android init for every process; read by bun_sys's runtime Android +// detection (`is_android_kernel`). +new!(pub ANDROID_DATA: string, "ANDROID_DATA", {}); +new!(pub ANDROID_ROOT: string, "ANDROID_ROOT", {}); new!(pub BUN_AGENT_RULE_DISABLED: boolean, "BUN_AGENT_RULE_DISABLED", { default: false }); new!(pub BUN_COMPILE_TARGET_TARBALL_URL: string, "BUN_COMPILE_TARGET_TARBALL_URL", {}); new!(pub BUN_CONFIG_DISABLE_COPY_FILE_RANGE: boolean, "BUN_CONFIG_DISABLE_COPY_FILE_RANGE", { default: false }); diff --git a/src/bun_core/fmt.rs b/src/bun_core/fmt.rs index 93c646200ed4..50a9835fe2c1 100644 --- a/src/bun_core/fmt.rs +++ b/src/bun_core/fmt.rs @@ -258,7 +258,7 @@ impl Display for RedactedNpmUrlFormatter<'_> { // Emit the run of bytes up to the next position where a uuid/npm // secret could possibly start, so multi-byte UTF-8 sequences are - // written intact (raw bytes, not Latin-1→UTF-8 chars). + // written intact (`BStr`: invalid UTF-8 becomes U+FFFD, not a bogus `&str`). let mut next = i + 1; while next < self.url.len() { let b = self.url[next]; @@ -271,7 +271,7 @@ impl Display for RedactedNpmUrlFormatter<'_> { } next += 1; } - write_bytes(f, &self.url[i..next])?; + write!(f, "{}", bstr::BStr::new(&self.url[i..next]))?; i = next; } Ok(()) @@ -282,6 +282,30 @@ pub fn redacted_npm_url(str: &[u8]) -> RedactedNpmUrlFormatter<'_> { RedactedNpmUrlFormatter { url: str } } +/// [`redacted_npm_url`] over a `Display` (a lockfile resolution, a dependency specifier); values +/// without `://` pass unchanged so a version whose pre-release tag looks like a token is not masked. +pub struct Redacted(T); + +impl Display for Redacted { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + // Rendered in full first: the password scan is anchored at the start of the whole value. + let text = self.0.to_string(); + if !strings::contains(text.as_bytes(), b"://") { + return f.write_str(&text); + } + redacted_npm_url(text.as_bytes()).fmt(f) + } +} + +pub fn redacted(value: T) -> Redacted { + Redacted(value) +} + +/// [`redacted`] + [`EscapeControlChars`]: how untrusted specifiers/resolutions go to a terminal. +pub fn for_terminal(value: T) -> EscapeControlChars> { + EscapeControlChars(Redacted(value)) +} + // ─────────────────────────────────────────────────────────────────────────── // RedactedSourceFormatter // ─────────────────────────────────────────────────────────────────────────── @@ -3326,6 +3350,70 @@ fn escape_powershell_impl(str: &[u8], writer: &mut impl fmt::Write) -> fmt::Resu write_bytes(writer, remain) } +// ─────────────────────────────────────────────────────────────────────────── +// escapeControlChars +// ─────────────────────────────────────────────────────────────────────────── + +/// Spells out C0/C1 controls and DEL (`\n`, `\x1b`, `\u009b`) in text somebody else authored +/// (registry manifests, a dependency's `package.json`) so it cannot repaint or forge our output. +pub struct EscapeControlChars(pub T); + +/// [`EscapeControlChars`] that lets `\t`, `\n` and `\r\n` through (`bun pm view readme`). +pub struct EscapeControlCharsMultiline(pub T); + +/// Invalid UTF-8 renders as U+FFFD. +pub fn escape_control_chars(text: &[u8]) -> EscapeControlChars<&bstr::BStr> { + EscapeControlChars(bstr::BStr::new(text)) +} + +pub fn escape_control_chars_multiline(text: &[u8]) -> EscapeControlCharsMultiline<&bstr::BStr> { + EscapeControlCharsMultiline(bstr::BStr::new(text)) +} + +impl Display for EscapeControlChars { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(EscapeControlCharsWriter(f, false), "{}", self.0) + } +} + +impl Display for EscapeControlCharsMultiline { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(EscapeControlCharsWriter(f, true), "{}", self.0) + } +} + +/// `.1`: keep `\t`, `\n`, `\r\n`. +struct EscapeControlCharsWriter<'a, 'f>(&'a mut Formatter<'f>, bool); + +impl fmt::Write for EscapeControlCharsWriter<'_, '_> { + fn write_str(&mut self, s: &str) -> fmt::Result { + let Self(f, keep_line_breaks) = self; + let mut start = 0; + let mut chars = s.char_indices().peekable(); + while let Some((i, c)) = chars.next() { + let pass_through = match c { + '\t' | '\n' => *keep_line_breaks, + '\r' => *keep_line_breaks && matches!(chars.peek(), Some((_, '\n'))), + '\0'..='\x1f' | '\x7f' | '\u{80}'..='\u{9f}' => false, + _ => true, + }; + if pass_through { + continue; + } + f.write_str(&s[start..i])?; + match c { + '\n' => f.write_str("\\n")?, + '\r' => f.write_str("\\r")?, + '\t' => f.write_str("\\t")?, + c if c.is_ascii() => write!(f, "\\x{:02x}", c as u32)?, + c => write!(f, "\\u{:04x}", c as u32)?, + } + start = i + c.len_utf8(); + } + f.write_str(&s[start..]) + } +} + // js_bindings (fmtString for highlighter.test.ts) lives in src/jsc/fmt_jsc.rs // alongside fmt_jsc.bind.ts; bun_core/ stays JSC-free. diff --git a/src/bun_core/lib.rs b/src/bun_core/lib.rs index 8428018adcc5..99193455be9d 100644 --- a/src/bun_core/lib.rs +++ b/src/bun_core/lib.rs @@ -2154,17 +2154,23 @@ pub(crate) mod strings_impl { /// Port of `bun.fmt.URLFormatter.findUrlPassword` — returns /// `(offset, len)` of the password segment, or None. - /// Only matches http:// and https:// schemes and rejects empty pw. + /// `s` must start with a URL (`https://`, `git+ssh://`, ...); rejects an + /// empty password. pub(crate) fn find_url_password(s: &[u8]) -> Option<(usize, usize)> { - // Case-sensitive prefix match; the search region is truncated at the - // first '\n' and at the end of the authority before scanning for '@'/':'. - let scheme_end = if s.starts_with(b"http://") { - 7 - } else if s.starts_with(b"https://") { - 8 - } else { + // RFC 3986 scheme (`git+https` is one scheme) followed by `://`; the + // search region is truncated at the first '\n' and at the end of the + // authority before scanning for '@'/':'. + if !s.first().is_some_and(u8::is_ascii_alphabetic) { return None; - }; + } + let scheme_len = s + .iter() + .take_while(|&&b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.')) + .count(); + if !s[scheme_len..].starts_with(b"://") { + return None; + } + let scheme_end = scheme_len + b"://".len(); let mut rest = &s[scheme_end..]; if let Some(nl) = crate::strings::index_of_char_usize(rest, b'\n') { rest = &rest[..nl]; diff --git a/src/bun_core/string/immutable.rs b/src/bun_core/string/immutable.rs index 46f64171aaf5..10abe3a41c91 100644 --- a/src/bun_core/string/immutable.rs +++ b/src/bun_core/string/immutable.rs @@ -967,6 +967,18 @@ pub fn is_utf8_char_boundary(c: u8) -> bool { (c as i8) >= -0x40 } +/// Longest prefix of `self_` within `max_len` bytes that does not split a UTF-8 sequence. +pub fn truncate_to_char_boundary(self_: &[u8], max_len: usize) -> &[u8] { + if self_.len() <= max_len { + return self_; + } + let mut end = max_len; + while !is_on_char_boundary(self_, end) { + end -= 1; + } + &self_[..end] +} + pub fn starts_with_case_insensitive_ascii(self_: &[u8], prefix: &[u8]) -> bool { self_.len() >= prefix.len() && eql_case_insensitive_ascii(&self_[0..prefix.len()], prefix, false) @@ -2722,6 +2734,22 @@ mod tests { assert!(!super::eql_case_insensitive_ascii(b"Ab", b"a", true)); } + #[test] + fn truncate_to_char_boundary_never_splits_a_sequence() { + assert_eq!(super::truncate_to_char_boundary(b"abc", 3), b"abc"); + assert_eq!(super::truncate_to_char_boundary(b"abc", 4), b"abc"); + assert_eq!(super::truncate_to_char_boundary(b"abcd", 3), b"abc"); + assert_eq!(super::truncate_to_char_boundary(b"abc", 0), b""); + // "aé" is `61 C3 A9`: a cut at byte 2 would land inside `é`. + assert_eq!(super::truncate_to_char_boundary("aéz".as_bytes(), 2), b"a"); + assert_eq!( + super::truncate_to_char_boundary("aéz".as_bytes(), 3), + "aé".as_bytes() + ); + // A 4-byte sequence that does not fit at all yields the empty prefix. + assert_eq!(super::truncate_to_char_boundary("😀".as_bytes(), 3), b""); + } + #[test] fn convert_utf8_to_utf16_in_buffer_fallback_rejects_malformed_sequences() { let mut buf = [0u16; 16]; diff --git a/src/dotenv/env_loader.rs b/src/dotenv/env_loader.rs index 57b4464602a8..1bb173f1cf9d 100644 --- a/src/dotenv/env_loader.rs +++ b/src/dotenv/env_loader.rs @@ -9,7 +9,7 @@ use bun_core::{self, Output}; use bun_core::{ZStr, strings}; use bun_paths::{self, MAX_PATH_BYTES, PathBuffer}; use bun_sys; -use bun_url::URL; +use bun_url::{OwnedURL, URL}; use bun_which::which; use enumset::EnumSet; @@ -322,7 +322,7 @@ impl Loader { result } - pub fn get_http_proxy_for(&self, url: &URL<'_>) -> Option> { + pub fn get_http_proxy_for(&self, url: &URL<'_>) -> Option { self.get_http_proxy(url.is_http(), Some(url.hostname), Some(url.host)) } @@ -336,32 +336,31 @@ impl Loader { /// Get proxy URL for HTTP/HTTPS requests, respecting NO_PROXY. /// `hostname` is the host without port (e.g., "localhost") /// `host` is the host with port if present (e.g., "localhost:3000") + /// Normalized like `fetch()`'s `proxy` option (#16182); a value WTF::URL rejects is used as is. pub fn get_http_proxy( &self, is_http: bool, hostname: Option<&[u8]>, host: Option<&[u8]>, - ) -> Option> { - // TODO: When Web Worker support is added, make sure to intern these strings - let mut http_proxy: Option> = None; - + ) -> Option { let proxy = if is_http { self.get_lower_then_upper(b"http_proxy", b"HTTP_PROXY") } else { self.get_lower_then_upper(b"https_proxy", b"HTTPS_PROXY") - }; - if let Some(p) = proxy { - if !Self::is_emptyish(p) { - http_proxy = Some(URL::parse(p)); - } + }?; + if Self::is_emptyish(proxy) || self.is_no_proxy(hostname, host) { + return None; } + Some( + URL::from_string(&bun_core::String::borrow_utf8(proxy)) + .unwrap_or_else(|_| OwnedURL::from_href(Box::from(proxy))), + ) + } - if http_proxy.is_some() && hostname.is_some() { - if self.is_no_proxy(hostname, host) { - return None; - } - } - http_proxy + /// The `no_proxy` / `NO_PROXY` list, or `None` when unset or empty-ish. + pub fn get_no_proxy(&self) -> Option<&[u8]> { + self.get_lower_then_upper(b"no_proxy", b"NO_PROXY") + .filter(|v| !Self::is_emptyish(v)) } /// Returns true if the given hostname/host should bypass the proxy @@ -371,12 +370,9 @@ impl Loader { // See the syntax at https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/ let Some(hn) = hostname else { return false }; - let Some(no_proxy_text) = self.get_lower_then_upper(b"no_proxy", b"NO_PROXY") else { + let Some(no_proxy_text) = self.get_no_proxy() else { return false; }; - if Self::is_emptyish(no_proxy_text) { - return false; - } for no_proxy_item in strings::split(no_proxy_text, b",") { let mut no_proxy_entry = strings::trim(no_proxy_item, &strings::WHITESPACE_CHARS); diff --git a/src/errno/lib.rs b/src/errno/lib.rs index 09e327f8bf72..ab0331eef9a0 100644 --- a/src/errno/lib.rs +++ b/src/errno/lib.rs @@ -304,6 +304,17 @@ pub fn from_errno(errno: i32) -> SystemErrno { SystemErrno::init(errno as i64).unwrap_or(SystemErrno::EIO) } +impl SystemErrno { + /// `None` if `err` is not an OS error or its code has no `SystemErrno`. + pub fn from_io_error(err: &std::io::Error) -> Option { + // A Win32 code there, which the `u32` entry point maps (`i64` would take it for an errno). + #[cfg(windows)] + return SystemErrno::init(err.raw_os_error()? as u32); + #[cfg(not(windows))] + return SystemErrno::init(i64::from(err.raw_os_error()?)); + } +} + #[cfg(not(windows))] impl SystemErrno { // `i64` covers every concrete call site (errno-range values). @@ -503,6 +514,21 @@ mod errno_name_tests { } } + #[test] + fn io_error_to_errno() { + let of = |code| SystemErrno::from_io_error(&std::io::Error::from_raw_os_error(code)); + #[cfg(not(windows))] + assert_eq!(of(libc::ENOMEM), Some(SystemErrno::ENOMEM)); + // Win32 ERROR_NOT_ENOUGH_MEMORY / ERROR_ACCESS_DENIED; as errno values 8 and 5 are ENOEXEC and EIO. + #[cfg(windows)] + assert_eq!( + (of(8), of(5)), + (Some(SystemErrno::ENOMEM), Some(SystemErrno::EPERM)) + ); + let not_os = std::io::Error::other("not from the OS"); + assert_eq!(SystemErrno::from_io_error(¬_os), None); + } + #[test] fn coreutils_map() { assert_eq!( diff --git a/src/http/HTTPThread.rs b/src/http/HTTPThread.rs index 912f7d4b903d..af39f0425871 100644 --- a/src/http/HTTPThread.rs +++ b/src/http/HTTPThread.rs @@ -5,6 +5,7 @@ use std::time::Instant; use bun_collections::ArrayHashMap; use bun_core::{self, Output}; +use bun_errno::SystemErrno; use bun_threading::{Mutex, UnboundedQueue}; use bun_uws as uws; @@ -1212,10 +1213,33 @@ mod _event_loop_draft { Ok(t) => { let _ = HTTP_THREAD_HANDLE.set(t); } - Err(err) => Output::panic(format_args!("Failed to start HTTP Client thread: {}", err)), + Err(err) => exit_spawn_failed(&err), } } + /// Nothing that needs the HTTP thread can go on without it, but a refused + /// `pthread_create`/`CreateThread` (`RLIMIT_NPROC`, a container pids limit, + /// no memory) is the environment's limit, not a bug: report it like any + /// other fatal CLI error rather than through the crash reporter. + #[cold] + #[inline(never)] + fn exit_spawn_failed(err: &std::io::Error) -> ! { + match SystemErrno::from_io_error(err) { + Some(errno) => { + bun_core::err_generic!("Failed to start HTTP Client thread: {}", errno); + if errno == SystemErrno::EAGAIN { + bun_core::note!( + "The process or thread limit may have been reached (ulimit -u, or the container's pids limit); raise it or reduce concurrency" + ); + } + } + // No errno name for this OS code (most Windows thread-creation + // failures): show the OS's own description instead of guessing one. + None => bun_core::err_generic!("Failed to start HTTP Client thread: {}", err), + } + bun_core::Global::crash() + } + fn on_start(opts: InitOpts) { Output::Source::configure_named_thread(bun_core::zstr!("HTTP Client")); diff --git a/src/http/lib.rs b/src/http/lib.rs index ab48559ff439..417013af4a1c 100644 --- a/src/http/lib.rs +++ b/src/http/lib.rs @@ -106,6 +106,17 @@ pub enum Protocol { Http3, } +/// Which redirects the request's credential headers survive. +#[repr(u8)] +#[derive(Copy, Clone, PartialEq, Eq, Default)] +pub enum RedirectCredentialsPolicy { + /// fetch(): https://fetch.spec.whatwg.org/#concept-http-redirect-fetch + #[default] + SameOrigin, + /// npm's rule, used by `bun install`: port and scheme changes keep them, except https -> http. + SameHostname, +} + pub use bun_http_types::Encoding::Encoding; pub use header_value_iterator::{ HeaderValueIterator, connection_header_keep_alive, upgrade_header_is_not_h2, @@ -214,6 +225,7 @@ pub struct Flags { pub forced_protocol: Option, pub(crate) h3_retried: bool, pub is_node_http_client: bool, + pub redirect_credentials: RedirectCredentialsPolicy, } impl Default for Flags { @@ -235,6 +247,7 @@ impl Default for Flags { forced_protocol: None, h3_retried: false, is_node_http_client: false, + redirect_credentials: RedirectCredentialsPolicy::SameOrigin, } } } @@ -662,54 +675,35 @@ pub struct ProxySettings { } impl ProxySettings { - /// Returns `None` when neither proxy is set: no re-evaluation is needed. - pub(crate) fn new( - http_proxy: Option<&[u8]>, - https_proxy: Option<&[u8]>, - no_proxy: Option<&[u8]>, + /// An empty href is "no proxy for that scheme"; `None` when neither is set. + fn new( + http_proxy: Box<[u8]>, + https_proxy: Box<[u8]>, + env: &bun_dotenv::Loader, ) -> Option> { - let http_proxy = http_proxy.unwrap_or(b""); - let https_proxy = https_proxy.unwrap_or(b""); if http_proxy.is_empty() && https_proxy.is_empty() { return None; } Some(Box::new(Self { - http_proxy: http_proxy.into(), - https_proxy: https_proxy.into(), - no_proxy: no_proxy.unwrap_or(b"").into(), + http_proxy, + https_proxy, + no_proxy: env.get_no_proxy().unwrap_or(b"").into(), })) } /// Capture `http_proxy` / `https_proxy` / `no_proxy` from the process env. pub fn from_env(env: &bun_dotenv::Loader) -> Option> { - #[inline] - fn is_emptyish(v: &[u8]) -> bool { - v.is_empty() || v == b"\"\"" || v == b"''" - } - // lowercase first; an empty lowercase value falls through to uppercase. - let read = |lower: &[u8], upper: &[u8]| -> Option<&[u8]> { - let v = env - .get(lower) - .filter(|v| !v.is_empty()) - .or_else(|| env.get(upper))?; - if is_emptyish(v) { None } else { Some(v) } + let proxy_href = |is_http: bool| -> Box<[u8]> { + env.get_http_proxy(is_http, None, None) + .map_or_else(Box::default, bun_url::OwnedURL::into_href) }; - Self::new( - read(b"http_proxy", b"HTTP_PROXY"), - read(b"https_proxy", b"HTTPS_PROXY"), - read(b"no_proxy", b"NO_PROXY"), - ) + Self::new(proxy_href(true), proxy_href(false), env) } /// Build from an explicit `fetch(url, { proxy })` option. The same proxy is /// used for both schemes; NO_PROXY is still consulted per hop. pub fn from_explicit(proxy_href: &[u8], env: &bun_dotenv::Loader) -> Option> { - let no_proxy = env - .get(b"no_proxy") - .filter(|v| !v.is_empty()) - .or_else(|| env.get(b"NO_PROXY")) - .filter(|v| !(v.is_empty() || *v == b"\"\"" || *v == b"''")); - Self::new(Some(proxy_href), Some(proxy_href), no_proxy) + Self::new(proxy_href.into(), proxy_href.into(), env) } /// Proxy href to use for `url`, or `None` for a direct connection. @@ -999,6 +993,7 @@ use bstr::BStr; use bun_boringssl as boringssl; use bun_collections::{ArrayHashMap, VecExt}; use bun_core::StringBuilder; +use bun_core::fmt::{EscapeControlChars, escape_control_chars}; use bun_core::{FeatureFlags, Global, Output}; use bun_core::{OwnedString, String as BunString, Tag as BunStringTag, strings}; use bun_http_types::ETag::StringPointer; @@ -1085,6 +1080,27 @@ bun_core::comptime_string_map! { }; } +#[derive(Copy, Clone)] +struct RedirectHop { + same_origin: bool, + /// What `RedirectCredentialsPolicy::SameHostname` keeps credentials on: never https -> http. + same_hostname_no_downgrade: bool, +} + +impl RedirectHop { + fn between(from: &URL<'_>, to: &URL<'_>) -> Self { + Self { + same_origin: strings::eql_case_insensitive_ascii( + strings::without_trailing_slash(to.origin), + strings::without_trailing_slash(from.origin), + true, + ), + same_hostname_no_downgrade: !(from.is_https() && !to.is_https()) + && strings::eql_case_insensitive_ascii(to.hostname, from.hostname, true), + } + } +} + // ── shared per-thread buffers ─────────────────────────────────────────── // All four are HTTP-thread-only scratch (single uws loop thread); `RacyCell` // is the alias-safe static cell per docs/PORTING.md §Global mutable state. @@ -1440,7 +1456,7 @@ pub(crate) fn print_request( "> {} {} {}", ver, BStr::new(request.method), - bun_core::fmt::redacted_npm_url(url), + EscapeControlChars(bun_core::fmt::redacted_npm_url(url)), ); for header in request.headers { let name = header.name(); @@ -1451,8 +1467,8 @@ pub(crate) fn print_request( let scheme_len = strings::index_of_char_usize(value, b' ').map_or(0, |i| i + 1); bun_core::pretty_errorln!( "> {}: {}[redacted]", - BStr::new(name), - BStr::new(&value[..scheme_len]), + escape_control_chars(name), + escape_control_chars(&value[..scheme_len]), ); } else { bun_core::pretty_errorln!("> {}", header); @@ -5021,7 +5037,7 @@ impl<'a> HTTPClient<'a> { { return Err(crate::Error::RequestBodyNotReusable); } - let is_same_origin; + let hop: RedirectHop; { if let Some(i) = strings::index_of(location, b"://") { @@ -5085,11 +5101,7 @@ impl<'a> HTTPClient<'a> { // `self.redirect` below, which lives as long as `self` (≥ `'a`). let new_url: URL<'a> = unsafe { URL::parse(&normalized_url_str).erase_lifetime() }; - is_same_origin = strings::eql_case_insensitive_ascii( - strings::without_trailing_slash(new_url.origin), - strings::without_trailing_slash(self.url.origin), - true, - ); + hop = RedirectHop::between(&self.url, &new_url); self.url = new_url; // connected_url still borrows from the previous hop's buffer // until doRedirect releases the socket, so park it in @@ -5140,11 +5152,7 @@ impl<'a> HTTPClient<'a> { // `self.redirect` below, which lives as long as `self` (≥ `'a`). let new_url: URL<'a> = unsafe { URL::parse(&normalized_url_str).erase_lifetime() }; - is_same_origin = strings::eql_case_insensitive_ascii( - strings::without_trailing_slash(new_url.origin), - strings::without_trailing_slash(self.url.origin), - true, - ); + hop = RedirectHop::between(&self.url, &new_url); self.url = new_url; debug_assert!(self.prev_redirect.is_empty()); self.prev_redirect = @@ -5168,11 +5176,7 @@ impl<'a> HTTPClient<'a> { // SAFETY: self-borrow — `new_url` is moved into `self.redirect` // below, which lives as long as `self` (≥ `'a`). self.url = unsafe { parsed_url.erase_lifetime() }; - is_same_origin = strings::eql_case_insensitive_ascii( - strings::without_trailing_slash(self.url.origin), - strings::without_trailing_slash(original_url.origin), - true, - ); + hop = RedirectHop::between(&original_url, &self.url); debug_assert!(self.prev_redirect.is_empty()); self.prev_redirect = core::mem::replace(&mut self.redirect, new_url); } @@ -5212,7 +5216,7 @@ impl<'a> HTTPClient<'a> { // Cross-origin redirect: re-derive SNI / cert // verification / Host from the redirect target. See // `InternalStateFlags::clear_hostname_on_redirect`. - if !is_same_origin { + if !hop.same_origin { self.state.flags.clear_hostname_on_redirect = true; } @@ -5221,13 +5225,19 @@ impl<'a> HTTPClient<'a> { // locationURL's origin, then for each headerName of CORS // non-wildcard request-header name, delete headerName from // request's header list. - if !is_same_origin && self.header_entries.len() > 0 { + // The credential headers (all but `host`) follow `Flags::redirect_credentials`. + if !hop.same_origin && self.header_entries.len() > 0 { + let strip_credentials = match self.flags.redirect_credentials { + RedirectCredentialsPolicy::SameOrigin => true, + RedirectCredentialsPolicy::SameHostname => !hop.same_hostname_no_downgrade, + }; let mut i = 0; while i < self.header_entries.len() { let name = self.header_str(self.header_entries.items_name()[i]); if CROSS_ORIGIN_STRIPPED_REQUEST_HEADERS .get_ascii_case_insensitive(name) .is_some() + && (strip_credentials || name.eq_ignore_ascii_case(b"host")) { let _ = self.header_entries.ordered_remove(i); } else { diff --git a/src/ini/lib.rs b/src/ini/lib.rs index fed27f26583d..d9df42d92dc6 100644 --- a/src/ini/lib.rs +++ b/src/ini/lib.rs @@ -131,7 +131,7 @@ mod draft { use core::ptr; use bun_alloc::{AllocError, Arena, ArenaVec, ArenaVecExt as _}; - use bun_api::{self, BunInstall, NpmRegistry, npm_registry}; + use bun_api::{self, BunInstall, NpmRegistry, NpmUrlAuth, npm_registry}; use bun_ast::E::Rope; use bun_ast::{E, Expr, ExprData, StoreRef}; use bun_ast::{Loc, Log, Source}; @@ -1183,6 +1183,22 @@ mod draft { RegistryCredential::Email(email) => registry.email.clone_from(email), } } + + /// Also record the line by `//host/path/` so it can be resolved by request URL (`NpmUrlAuth`). + fn apply_to_url_auth(&self, url_auth: &mut Vec) { + let existing = url_auth + .iter() + .position(|entry| entry.host == self.host && entry.pathname == self.pathname); + let index = existing.unwrap_or_else(|| { + url_auth.push(NpmUrlAuth { + host: self.host.clone(), + pathname: self.pathname.clone(), + credentials: NpmRegistry::default(), + }); + url_auth.len() - 1 + }); + self.apply_to(&mut url_auth[index].credentials); + } } // ────────────────────────────────────────────────────────────────────────── @@ -1594,6 +1610,7 @@ mod draft { continue; } if let Some(auth) = RegistryAuth::from_config_item(conf_item, iter.log, source) { + auth.apply_to_url_auth(&mut install.url_auth); configs.push(auth); } } diff --git a/src/install/NetworkTask.rs b/src/install/NetworkTask.rs index b7bda6f5613d..5e90e7d57d0e 100644 --- a/src/install/NetworkTask.rs +++ b/src/install/NetworkTask.rs @@ -4,14 +4,15 @@ use core::sync::atomic::Ordering; use crate::bun_fs::{FileSystem, FilenameStore}; use bun_collections::HashMap; -use bun_core::{self, fmt::quote}; -use bun_core::{MutableString, strings}; +use bun_core::fmt::{quote, redacted_npm_url}; +use bun_core::{self, MutableString, strings}; use bun_http::{ self as http, AsyncHTTP, HTTPClientResult, HTTPClientResultCallback, HTTPVerboseLevel, HeaderBuilder, async_http::Options as AsyncHTTPOptions, }; use bun_threading::thread_pool::Batch; -use bun_url::URL; +use bun_url::{OwnedURL, URL}; +use std::io::Write as _; use crate::extract_tarball; use crate::npm::{self as npm, PackageManifest}; @@ -61,6 +62,10 @@ pub struct NetworkTask { // into `callback`; owning avoids that at the cost of one copy per tarball download. pub(crate) url_buf: Box<[u8]>, pub(crate) header_buf: Box<[u8]>, + /// Proxy href for this request (empty: direct); owned like `url_buf`, `AsyncHTTP` borrows it. + pub(crate) http_proxy_buf: Box<[u8]>, + /// Read back when a 401/403 is reported (`Options::missing_credentials_note`). + pub(crate) authorization: Authorization, pub(crate) retried: u16, pub(crate) response_buffer: MutableString, // BACKREF: PackageManager owns this task via `preallocated_network_tasks`. @@ -125,6 +130,21 @@ pub struct DedupeMapEntry { /// later `enqueue_*_for_download` can observe the failure instead of /// re-scheduling the entire network task (and its retry cycle) a second time. pub(crate) failed: bool, + /// Manifest tasks only (these three): see `PackageManager::has_created_manifest_task`. + pub(crate) is_extended_manifest: bool, + pub(crate) has_abbreviated_manifest_request: bool, + /// The package then resolves from the abbreviated document (`PackageManager::needs_extended_manifest`). + pub(crate) extended_manifest_failed: bool, +} + +impl DedupeMapEntry { + pub(crate) fn manifest_requested(&mut self, extended: bool) -> &mut bool { + if extended { + &mut self.is_extended_manifest + } else { + &mut self.has_abbreviated_manifest_request + } + } } /// `Id` is already a wyhash output, so identity hashing /// (hash = value bits) avoids re-hashing. @@ -171,6 +191,21 @@ impl NetworkTask { unsafe { self.package_manager.assume_mut() } } + /// Stores the proxy for `url` in `http_proxy_buf` and returns the view the `AsyncHTTP` keeps. + fn http_proxy_for(&mut self, pm: &PackageManager, url: &URL<'_>) -> Option> { + self.http_proxy_buf = pm + .http_proxy(url) + .map_or_else(Box::default, OwnedURL::into_href); + if self.http_proxy_buf.is_empty() { + return None; + } + // SAFETY: same lifetime extension as `url_buf` in `for_manifest`: `run_tasks` drops + // `unsafe_http_client` before the slot (and this buffer) goes back to the pool. + Some(URL::parse(unsafe { + bun_ptr::detach_lifetime(&self.http_proxy_buf) + })) + } + // Signature matches `HTTPClientResultCallback::new::`'s // `fn(*mut T, *mut AsyncHTTP, HTTPClientResult<'_>)` shape so it can be // installed directly without a separate trampoline. @@ -408,31 +443,28 @@ fn count_auth(header_builder: &mut HeaderBuilder, scope: &npm::registry::Scope) header_builder.count("npm-auth-type", "legacy"); } -/// Splits `http://user:pass@host/pkg.tgz` into `user:pass` and `http://host/pkg.tgz`; only an `@` in the authority counts, not `/@scope/`. +/// `http://user:pass@host/pkg.tgz` -> (`user:pass`, `http://host/pkg.tgz`). Like `bun_url`, userinfo +/// ends at the last `@` before the first `/` (so `#`/`?` in a password stay, `/@scope/` is path). fn split_url_userinfo(url: &[u8]) -> Option<(&[u8], Box<[u8]>)> { let authority_start = strings::index_of(url, b"://")? + b"://".len(); let rest = &url[authority_start..]; - let authority = &rest[..strings::index_of_any(rest, b"/?#").unwrap_or(rest.len())]; + let authority = &rest[..strings::index_of_char_usize(rest, b'/').unwrap_or(rest.len())]; let at = strings::last_index_of_char(authority, b'@')?; - - let mut without_userinfo = Vec::with_capacity(url.len() - (at + 1)); - without_userinfo.extend_from_slice(&url[..authority_start]); - without_userinfo.extend_from_slice(&rest[at + 1..]); + let without_userinfo = [&url[..authority_start], &rest[at + 1..]].concat(); Some((&rest[..at], without_userinfo.into_boxed_slice())) } -/// `Basic base64(userinfo)` as written (no percent-decoding, `user` means `user:`), matching what npm sends via node's `auth` option. +/// `Basic base64(userinfo)` as npm sends it: not percent-decoded, and no `:` means an empty password. fn basic_authorization_from_userinfo(userinfo: &[u8]) -> Vec { - const SCHEME: &[u8] = b"Basic "; - let mut user_pass = Vec::with_capacity(userinfo.len() + 1); - user_pass.extend_from_slice(userinfo); + let mut user_pass = userinfo.to_vec(); if !strings::contains_char(userinfo, b':') { user_pass.push(b':'); } - let mut value = vec![0u8; SCHEME.len() + bun_core::base64::encode_len(&user_pass)]; - value[..SCHEME.len()].copy_from_slice(SCHEME); - let encoded_len = bun_core::base64::encode(&mut value[SCHEME.len()..], &user_pass); - value.truncate(SCHEME.len() + encoded_len); + let mut value = b"Basic ".to_vec(); + let start = value.len(); + value.resize(start + bun_core::base64::encode_len(&user_pass), 0); + let encoded_len = bun_core::base64::encode(&mut value[start..], &user_pass); + value.truncate(start + encoded_len); value } @@ -463,6 +495,13 @@ impl bun_core::output::ErrName for ForManifestError { } } +/// Redacted before `quote()`: the password scan only recognizes an unquoted URL. +fn redacted_url(url: &[u8]) -> Vec { + let mut out: Vec = Vec::new(); + let _ = write!(out, "{}", redacted_npm_url(url)); + out +} + impl NetworkTask { pub(crate) fn for_manifest( &mut self, @@ -499,13 +538,14 @@ impl NetworkTask { )); if tmp.tag() == bun_core::Tag::Dead { + let redacted_registry = redacted_url(scope.url.href()); if !is_optional { log.add_error_fmt( None, bun_ast::Loc::EMPTY, format_args!( "Failed to join registry {} and package {} URLs", - quote(scope.url.href()), + quote(&redacted_registry), quote(name), ), ); @@ -515,7 +555,7 @@ impl NetworkTask { bun_ast::Loc::EMPTY, format_args!( "Failed to join registry {} and package {} URLs", - quote(scope.url.href()), + quote(&redacted_registry), quote(name), ), ); @@ -523,14 +563,18 @@ impl NetworkTask { return Err(ForManifestError::InvalidURL); } + // This actually duplicates the string! So we defer deref the WTF managed one above. + let url_bytes = tmp.to_owned_slice().into_boxed_slice(); + if !(tmp.has_prefix_comptime(b"https://") || tmp.has_prefix_comptime(b"http://")) { + let redacted_manifest_url = redacted_url(&url_bytes); if !is_optional { log.add_error_fmt( None, bun_ast::Loc::EMPTY, format_args!( - "Registry URL must be http:// or https://\nReceived: \"{}\"", - *tmp + "Registry URL must be http:// or https://\nReceived: {}", + quote(&redacted_manifest_url) ), ); } else { @@ -538,17 +582,14 @@ impl NetworkTask { None, bun_ast::Loc::EMPTY, format_args!( - "Registry URL must be http:// or https://\nReceived: \"{}\"", - *tmp + "Registry URL must be http:// or https://\nReceived: {}", + quote(&redacted_manifest_url) ), ); } return Err(ForManifestError::InvalidURL); } - // This actually duplicates the string! So we defer deref the WTF managed one above. - let url_bytes = tmp.to_owned_slice().into_boxed_slice(); - { let joined = URL::parse(&url_bytes); let registry = scope.url.url(); @@ -560,6 +601,8 @@ impl NetworkTask { || joined.get_port_auto() != registry.get_port_auto() || !joined.pathname.starts_with(registry_dir) { + let redacted_manifest_url = redacted_url(&url_bytes); + let redacted_registry = redacted_url(scope.url.href()); if !is_optional { log.add_error_fmt( None, @@ -567,8 +610,8 @@ impl NetworkTask { format_args!( "Invalid package name {}: manifest URL {} is not on registry {}", quote(name), - quote(&url_bytes), - quote(scope.url.href()), + quote(&redacted_manifest_url), + quote(&redacted_registry), ), ); } else { @@ -578,8 +621,8 @@ impl NetworkTask { format_args!( "Invalid package name {}: manifest URL {} is not on registry {}", quote(name), - quote(&url_bytes), - quote(scope.url.href()), + quote(&redacted_manifest_url), + quote(&redacted_registry), ), ); } @@ -599,6 +642,8 @@ impl NetworkTask { } } + self.authorization = Authorization::AllowAuthorization; + let mut header_builder = HeaderBuilder::default(); count_auth(&mut header_builder, scope); @@ -670,7 +715,7 @@ impl NetworkTask { // SAFETY: `self.url_buf` outlives the request, same as `header_buf` above (see `s3/simple_request.rs`). let url = URL::parse(unsafe { bun_ptr::detach_lifetime(&self.url_buf) }); - let http_proxy = pm.http_proxy(&url); + let http_proxy = self.http_proxy_for(pm, &url); let completion_callback = self.get_completion_callback(); // MaybeUninit overwrite — see field doc; old slot value is // either uninitialized (fresh hive slot) or a stale bitwise copy from @@ -689,6 +734,8 @@ impl NetworkTask { }, )); self.http_mut().client.flags.reject_unauthorized = pm.tls_reject_unauthorized(); + self.http_mut().client.flags.redirect_credentials = + http::RedirectCredentialsPolicy::SameHostname; if PackageManager::verbose_install() { self.http_mut().client.verbose = HTTPVerboseLevel::Headers; @@ -798,6 +845,7 @@ impl NetworkTask { }; if !(self.url_buf.starts_with(b"https://") || self.url_buf.starts_with(b"http://")) { + let redacted_tarball_url = redacted_url(&self.url_buf); // SAFETY: `pm.log` is the long-lived `*mut Log` the package // manager was constructed with. pm.log_mut().add_error_fmt( @@ -805,69 +853,48 @@ impl NetworkTask { bun_ast::Loc::EMPTY, format_args!( "Expected tarball URL to start with https:// or http://, got {} while fetching package {}", - quote(&self.url_buf), + quote(&redacted_tarball_url), quote(tarball.name.slice()), ), ); return Err(ForTarballError::InvalidURL); } - // Userinfo becomes a header and leaves the URL: `bun_url` keeps it in `origin`, which the redirect same-origin check compares. - let url_authorization: Option> = match split_url_userinfo(&self.url_buf) { - Some((userinfo, url_without_userinfo)) => { - let value = - (!userinfo.is_empty()).then(|| basic_authorization_from_userinfo(userinfo)); - self.url_buf = url_without_userinfo; - value - } - None => None, - }; + // `https://user:pass@host/dep.tgz`: sent as a header, as npm does; left in the URL it would + // be part of `origin` to `bun_url` and a same-host redirect would drop `Authorization`. + let mut url_authorization: Option> = None; + if let Some((userinfo, url_without_userinfo)) = split_url_userinfo(&self.url_buf) { + url_authorization = + (!userinfo.is_empty()).then(|| basic_authorization_from_userinfo(userinfo)); + self.url_buf = url_without_userinfo; + } - // Only attach the registry `Authorization` header when the tarball URL - // origin matches the configured registry scope origin. The npm manifest - // is registry-controlled, so a malicious registry could otherwise point - // the tarball at an attacker-controlled host and receive the scope - // credentials. The empty-`tarball_url` branch builds the URL from - // `scope.url.href()`, so its origin matches and authorized downloads - // keep working. - // Compare (protocol, hostname, effective port) rather than the raw - // `URL.origin` slice — `origin` is a borrowed prefix of the input - // string and is not normalized for default ports, so a tarball URL of - // `https://host:443/...` would not byte-match a `.npmrc` registry of - // `https://host/...` even though they are the same origin. Some - // registries emit `dist.tarball` URLs with the default port spelled - // out; without normalization those installs lose the `Authorization` - // header and fail with 401. - let send_auth = matches!(authorization, Authorization::AllowAuthorization) && { - let tarball = URL::parse(&self.url_buf); - let registry = scope.url.url(); - tarball.protocol == registry.protocol - && tarball.hostname == registry.hostname - && tarball.get_port_auto() == registry.get_port_auto() + self.authorization = authorization; + let credentials = match authorization { + Authorization::NoAuthorization => None, + Authorization::AllowAuthorization => pm + .options + .tarball_credentials(scope, &URL::parse(&self.url_buf)), }; + // As in npm, credentials configured for the host win over the ones embedded in the URL. + let url_authorization = url_authorization.filter(|_| credentials.is_none()); self.response_buffer = MutableString::init_empty(); let mut header_builder = HeaderBuilder::default(); - if send_auth { - count_auth(&mut header_builder, scope); + if let Some(value) = &url_authorization { + header_builder.count("Authorization", value); + } else if let Some(credentials) = credentials { + count_auth(&mut header_builder, credentials); } - // Registry credentials win over URL userinfo, as in npm. - let url_authorization = match url_authorization { - Some(value) if header_builder.header_count == 0 => { - header_builder.count("Authorization", &value); - Some(value) - } - _ => None, - }; - let header_buf: &'static [u8] = if header_builder.header_count > 0 { header_builder.allocate()?; - match &url_authorization { - Some(value) => header_builder.append("Authorization", value), - None => append_auth(&mut header_builder, scope), + if let Some(value) = &url_authorization { + header_builder.append("Authorization", value); + } else if let Some(credentials) = credentials { + append_auth(&mut header_builder, credentials); } debug_assert_eq!(header_builder.content.len, header_builder.content.cap); self.header_buf = header_builder.content.move_to_slice(); @@ -885,7 +912,7 @@ impl NetworkTask { let url = URL::parse(unsafe { bun_ptr::detach_lifetime(&self.url_buf) }); let mut http_options = AsyncHTTPOptions { - http_proxy: pm.http_proxy(&url), + http_proxy: self.http_proxy_for(pm, &url), ..Default::default() }; @@ -934,6 +961,8 @@ impl NetworkTask { http_options, )); self.http_mut().client.flags.reject_unauthorized = pm.tls_reject_unauthorized(); + self.http_mut().client.flags.redirect_credentials = + http::RedirectCredentialsPolicy::SameHostname; if PackageManager::verbose_install() { self.http_mut().client.verbose = HTTPVerboseLevel::Headers; } @@ -1018,6 +1047,8 @@ impl NetworkTask { addr_of_mut!((*slot).response).write(HTTPClientResult::default()); addr_of_mut!((*slot).url_buf).write(Box::default()); addr_of_mut!((*slot).header_buf).write(Box::default()); + addr_of_mut!((*slot).http_proxy_buf).write(Box::default()); + addr_of_mut!((*slot).authorization).write(Authorization::NoAuthorization); addr_of_mut!((*slot).retried).write(0); addr_of_mut!((*slot).next).write(bun_threading::Link::new()); addr_of_mut!((*slot).tarball_stream).write(None); diff --git a/src/install/PackageInstall.rs b/src/install/PackageInstall.rs index 6adf994d1ad9..bc3c9bf376a7 100644 --- a/src/install/PackageInstall.rs +++ b/src/install/PackageInstall.rs @@ -5,7 +5,7 @@ use bun_core::Progress::Progress; use bun_core::{Global, Output}; use bun_core::{MutableString, ZStr}; use bun_paths::strings; -use bun_paths::{self as path, OSPathChar, OSPathSlice, PathBuffer, SEP, SEP_STR}; +use bun_paths::{self as path, OSPathChar, OSPathSlice, PathBuffer}; use bun_semver::String as SemverString; #[cfg(not(windows))] use bun_sys::OpenDirOptions; @@ -13,7 +13,7 @@ use bun_sys::{self as sys, Dir, EntryKind, Fd, FdExt, walker_skippable}; use bun_threading::thread_pool::{Batch, Node as ThreadPoolNode}; use bun_threading::work_pool::Task as WorkPoolTask; #[cfg(windows)] -use bun_threading::{ThreadPool, WaitGroup}; +use bun_threading::{IntrusiveWorkTask, ThreadPool, WaitGroup}; use crate::package_installer::NodeModulesFolder; use crate::{ @@ -29,10 +29,7 @@ pub struct PackageInstall<'a> { /// short-lived `Dir` held by the caller — `PackageInstall` never closes it. pub(crate) cache_dir: Fd, pub(crate) cache_dir_subpath: &'a ZStr, - // TODO: `destination_dir_subpath` aliases into `destination_dir_subpath_buf`; - // borrowck will reject simultaneous &ZStr + &mut [u8]. Consider storing only the len. pub(crate) destination_dir_subpath: &'a ZStr, - pub(crate) destination_dir_subpath_buf: &'a mut [u8], pub(crate) progress: Option<&'a mut Progress>, @@ -234,6 +231,7 @@ pub enum Step { OpeningCacheDir, OpeningDestDir, CopyingFiles, + MovingIntoPlace, LinkingDependency, } @@ -244,11 +242,58 @@ impl Step { Step::CopyingFiles => b"copying files from cache to destination", Step::OpeningCacheDir => b"opening cache/package/version dir", Step::OpeningDestDir => b"opening node_modules/package dir", + Step::MovingIntoPlace => b"moving copied files into node_modules/package dir", Step::LinkingDependency => b"linking dependency/workspace to node_modules", } } } +/// Where a package is linked to before being renamed onto its real path, which +/// later installs take as proof that it is installed: `@scope/name` becomes +/// `@scope/.bun-tmp-`. Hashed because a name may already be NAME_MAX long, +/// deterministic so the next install of the package removes a stale one. +pub(crate) struct StagingPath<'a>(pub(crate) &'a [u8]); + +impl core::fmt::Display for StagingPath<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let name_start = strings::last_index_of_char(self.0, b'/').map_or(0, |slash| slash + 1); + let scope = &self.0[..name_start]; + write!( + f, + "{}.bun-tmp-{:016x}", + bstr::BStr::new(scope), + bun_wyhash::hash(self.0) + ) + } +} + +/// Renames a fully linked `StagingPath` (relative to `dir`) onto `dest`. Fails if +/// `dest` is occupied. +pub(crate) fn rename_staging_into_place(dir: Fd, staging: &ZStr, dest: &ZStr) -> sys::Maybe<()> { + #[cfg(windows)] + { + // A scanner still holding a just-written file open fails the rename for a + // few milliseconds (#11250 is the same failure for the cache). An occupied + // `dest` fails with the same errors and is not worth waiting on. + const RETRIES: u32 = 6; + for attempt in 0..RETRIES { + match sys::renameat(dir, staging, dir, dest) { + Err(err) + if matches!( + err.get_errno(), + sys::E::EPERM | sys::E::EACCES | sys::E::EBUSY + ) && !sys::directory_exists_at(dir, dest).unwrap_or(false) => + { + // 10ms, 20ms, ... 320ms: 630ms in total. + std::thread::sleep(std::time::Duration::from_millis(10u64 << attempt)); + } + result => return result, + } + } + } + sys::renameat(dir, staging, dir, dest) +} + // PORTING.md §Global mutable state: install-main-thread enum. `RacyCell` // (no `Atomic`) — writers are the CLI option-load and the // clonefile/hardlink fallback in `install_with_method`, all on the install @@ -450,18 +495,16 @@ impl NewTaskQueue { } /// # Safety - /// `task` must point to a live, Box-allocated `TaskType` whose ownership is - /// being handed to the thread pool; the worker reclaims it in its callback. + /// `task` must be the `heap::into_raw` pointer of a live `TaskType` whose + /// ownership is being handed to the thread pool; the worker reclaims it. pub(crate) unsafe fn push(&self, task: *mut TaskType) where - TaskType: HasWorkPoolTask, + TaskType: IntrusiveWorkTask, { self.wait_group.add_one(); - // SAFETY: caller contract — `task` is a valid Box-allocated task; `.task()` - // is the intrusive node field. - self.thread_pool.schedule(Batch::from(unsafe { - std::ptr::from_mut::((*task).task()) - })); + // SAFETY: caller contract; projecting through `task` keeps the allocation's provenance. + self.thread_pool + .schedule(Batch::from(unsafe { TaskType::field_of(task) })); } pub(crate) fn wait(&self) { @@ -469,11 +512,6 @@ impl NewTaskQueue { } } -#[cfg(windows)] -pub(crate) trait HasWorkPoolTask { - fn task(&mut self) -> &mut WorkPoolTask; -} - // ───────────────────────────── HardLinkWindowsInstallTask ───────────────────────────── #[cfg(windows)] @@ -490,11 +528,7 @@ struct HardLinkWindowsInstallTask { } #[cfg(windows)] -impl HasWorkPoolTask for HardLinkWindowsInstallTask { - fn task(&mut self) -> &mut WorkPoolTask { - &mut self.task - } -} +bun_threading::intrusive_work_task!(HardLinkWindowsInstallTask, task); #[cfg(windows)] type HardLinkQueue = NewTaskQueue; @@ -569,8 +603,8 @@ impl HardLinkWindowsInstallTask { } fn run_from_thread_pool(task: *mut WorkPoolTask) { - // SAFETY: task points to the `task` field of a HardLinkWindowsInstallTask. - let self_: *mut Self = unsafe { bun_core::from_field_ptr!(Self, task, task) }; + // SAFETY: `task` is the `task` field pointer `NewTaskQueue::push` scheduled. + let self_: *mut Self = unsafe { Self::from_task_ptr(task) }; // SAFETY: HARDLINK_QUEUE initialized by init_queue() before scheduling. let queue = unsafe { (*HARDLINK_QUEUE.get()).assume_init_ref() }; scopeguard::defer! { queue.complete_one(); } @@ -778,23 +812,10 @@ impl<'a> PackageInstall<'a> { // 1. verify that .bun-tag exists (was it installed from bun?) // 2. check .bun-tag against the resolved version fn verify_git_resolution(&mut self, repo: &Repository, root_node_modules_dir: &Dir) -> bool { - let dest_len = self.destination_dir_subpath.len(); - let suffix: &[u8] = &[SEP, b'.', b'b', b'u', b'n', b'-', b't', b'a', b'g']; - // Reshaped for borrowck — write into buf via raw indices. - self.destination_dir_subpath_buf[dest_len..dest_len + suffix.len()].copy_from_slice(suffix); - self.destination_dir_subpath_buf[dest_len + SEP_STR.len() + b".bun-tag".len()] = 0; - // SAFETY: NUL written above. - let bun_tag_path = unsafe { - ZStr::from_raw_mut( - self.destination_dir_subpath_buf.as_mut_ptr(), - dest_len + SEP_STR.len() + b".bun-tag".len(), - ) - }; - let _restore = scopeguard::guard( - self.destination_dir_subpath_buf.as_mut_ptr(), - // SAFETY: p points into destination_dir_subpath_buf which outlives this scope; - // dest_len < buf capacity (was the prior NUL position). - move |p| unsafe { *p.add(dest_len) = 0 }, + let mut spill = Vec::new(); + let bun_tag_path = path::resolve_path::join_z_spill::( + &mut spill, + &[self.destination_dir_subpath.as_bytes(), b".bun-tag"], ); let Ok(bun_tag_file) = self @@ -859,25 +880,10 @@ impl<'a> PackageInstall<'a> { mutable.reset(); mutable.expand_to_capacity(); - let dest_len = self.destination_dir_subpath.len(); - // Write the literal directly into the path buffer; no intermediate Vec. - let suffix: &[u8] = &[ - SEP, b'p', b'a', b'c', b'k', b'a', b'g', b'e', b'.', b'j', b's', b'o', b'n', - ]; - self.destination_dir_subpath_buf[dest_len..dest_len + suffix.len()].copy_from_slice(suffix); - self.destination_dir_subpath_buf[dest_len + SEP_STR.len() + b"package.json".len()] = 0; - // SAFETY: NUL written above. - let package_json_path = unsafe { - ZStr::from_raw_mut( - self.destination_dir_subpath_buf.as_mut_ptr(), - dest_len + SEP_STR.len() + b"package.json".len(), - ) - }; - let _restore = scopeguard::guard( - self.destination_dir_subpath_buf.as_mut_ptr(), - // SAFETY: p points into destination_dir_subpath_buf which outlives this scope; - // dest_len < buf capacity (was the prior NUL position). - move |p| unsafe { *p.add(dest_len) = 0 }, + let mut spill = Vec::new(); + let package_json_path = path::resolve_path::join_z_spill::( + &mut spill, + &[self.destination_dir_subpath.as_bytes(), b"package.json"], ); let package_json_file = self @@ -1091,13 +1097,9 @@ impl<'a> PackageInstall<'a> { #[cfg(target_os = "macos")] fn install_with_clonefile(&mut self, destination_dir: &Dir) -> crate::Result { if self.destination_dir_subpath.as_bytes()[0] == b'@' { - if let Some(slash) = strings::index_of_char_z(self.destination_dir_subpath, SEP) { - let slash = slash as usize; - self.destination_dir_subpath_buf[slash] = 0; - // SAFETY: NUL written above. - let subdir = ZStr::from_buf(self.destination_dir_subpath_buf, slash); - let _ = sys::mkdirat(destination_dir, subdir, 0o755); - self.destination_dir_subpath_buf[slash] = SEP; + if let Some(slash) = strings::index_of_char_z(self.destination_dir_subpath, path::SEP) { + let subdir = &self.destination_dir_subpath.as_bytes()[..slash as usize]; + let _ = destination_dir.make_dir(subdir); } } @@ -1733,8 +1735,8 @@ impl<'a> PackageInstall<'a> { { let cache_dir_path = sys::get_fd_path(state.walker.root(), &mut buf2)?; let cache_len = cache_dir_path.len(); - if cache_len > 0 && cache_dir_path[cache_len - 1] != SEP { - buf2[cache_len] = SEP; + if cache_len > 0 && cache_dir_path[cache_len - 1] != path::SEP { + buf2[cache_len] = path::SEP; to_copy_buf2_offset = cache_len + 1; } else { to_copy_buf2_offset = cache_len; @@ -2094,7 +2096,11 @@ impl<'a> PackageInstall<'a> { } } }; - let dest = bun_paths::basename(dest_path.as_bytes()); + // The entry name is the NUL-terminated tail of `dest_path`. + let dest: &ZStr = ZStr::from_slice_with_nul( + &dest_path.as_bytes_with_nul()[subdir.map_or(0, |dir| dir.len() + 1)..], + ); + debug_assert_eq!(dest.as_bytes(), bun_paths::basename(dest_path.as_bytes())); // When we're linking on Windows, we want to avoid keeping the source directory handle open #[cfg(windows)] { @@ -2145,7 +2151,14 @@ impl<'a> PackageInstall<'a> { dest_buf[offset] = bun_paths::SEP_WINDOWS; offset += 1; } - dest_buf[offset..offset + dest.len()].copy_from_slice(dest); + if offset + dest.len() >= dest_buf.len() { + return InstallResult::fail( + crate::Error::Sys(bun_errno::SystemErrno::ENAMETOOLONG), + Step::LinkingDependency, + None, + ); + } + dest_buf[offset..offset + dest.len()].copy_from_slice(dest.as_bytes()); offset += dest.len(); dest_buf[offset] = 0; @@ -2204,19 +2217,13 @@ impl<'a> PackageInstall<'a> { Err(err) => return InstallResult::fail(err.into(), Step::LinkingDependency, None), }; - let target = path::resolve_path::relative(dest_dir_path, to_path); - // `symlinkat` takes `&ZStr` for both target and dest; build NUL-terminated - // copies in stack buffers. let mut target_buf = PathBuffer::uninit(); - target_buf[..target.len()].copy_from_slice(target); - target_buf[target.len()] = 0; - // SAFETY: NUL written above. - let target_z = ZStr::from_buf(&target_buf, target.len()); - let mut dest_name_buf = [0u8; 512]; - dest_name_buf[..dest.len()].copy_from_slice(dest); - // SAFETY: zero-initialized; NUL at [dest.len()]. - let dest_z = ZStr::from_buf(&dest_name_buf, dest.len()); - if let Err(err) = sys::symlinkat(target_z, dest_dir.fd(), dest_z) { + let target = path::resolve_path::relative_buf_z( + target_buf.as_mut_slice(), + dest_dir_path, + to_path, + ); + if let Err(err) = sys::symlinkat(target, dest_dir.fd(), dest) { return InstallResult::fail(err.into(), Step::LinkingDependency, None); } } @@ -2250,6 +2257,8 @@ impl<'a> PackageInstall<'a> { ) -> bool { let state = manager.get_preinstall_state(package_id); match state { + // no entry name until extraction records a local tarball's integrity + _ if self.cache_dir_subpath.is_empty() => true, crate::PreinstallState::Done => false, _ => { let exists = if self.patch.is_none() { @@ -2308,6 +2317,56 @@ impl<'a> PackageInstall<'a> { self.uninstall_before_install(destination_dir); } + let mut staging_buf = path::path_buffer_pool::get(); + let Ok(staging) = bun_core::fmt::buf_print_z( + &mut staging_buf[..], + format_args!("{}", StagingPath(self.destination_dir_subpath.as_bytes())), + ) else { + return InstallResult::fail( + crate::Error::Sys(bun_errno::SystemErrno::ENAMETOOLONG), + Step::OpeningDestDir, + None, + ); + }; + // A stale one may hold files of another version, which the backends would keep. + if let Err(err) = destination_dir.delete_tree(staging.as_bytes()) { + return InstallResult::fail(err.into(), Step::OpeningDestDir, None); + } + + let staged = PackageInstall { + destination_dir_subpath: staging, + progress: self.progress.as_deref_mut(), + ..*self + } + .install_into(destination_dir, method_, resolution_tag); + if let failure @ InstallResult::Failure(_) = staged { + let _ = destination_dir.delete_tree(staging.as_bytes()); + return failure; + } + + let dest = self.destination_dir_subpath; + let mut renamed = rename_staging_into_place(destination_dir.fd(), staging, dest); + if renamed.is_err() && dest.as_bytes() != b"." { + // Occupied: a workspace depended on under two names is walked as two trees, + // so the packages inside it are installed twice. The later one replaces it. + self.uninstall_before_install(destination_dir); + renamed = rename_staging_into_place(destination_dir.fd(), staging, dest); + } + match renamed { + Ok(()) => InstallResult::Success, + Err(err) => { + let _ = destination_dir.delete_tree(staging.as_bytes()); + InstallResult::fail(err.into(), Step::MovingIntoPlace, None) + } + } + } + + fn install_into( + &mut self, + destination_dir: &Dir, + method_: Method, + resolution_tag: resolution::Tag, + ) -> InstallResult { let mut supported_method_to_use = method_; if resolution_tag == resolution::Tag::Folder diff --git a/src/install/PackageInstaller.rs b/src/install/PackageInstaller.rs index 6c96867f593b..a157fd2aa722 100644 --- a/src/install/PackageInstaller.rs +++ b/src/install/PackageInstaller.rs @@ -288,6 +288,11 @@ pub struct TreeContext { /// Number of installed dependencies. Could be successful or failure. pub(crate) install_count: usize, + + /// Dependencies whose folder in this tree is replaced by this install. + pub(crate) replaced: Vec, + /// Inside an ancestor's `replaced` folder: what is on disk here goes away, so never skip. + pub(crate) inside_replaced_folder: bool, } type TreeContextId = lockfile::tree::Id; @@ -403,24 +408,15 @@ impl<'a> LazyPackageDestinationDir<'a> { } } -/// A dependency alias becomes the install destination inside `node_modules` -/// (the existing entry is renamed aside, deleted, and re-created). Reject -/// anything that could escape `node_modules`: empty names, `.`/`..` -/// components, absolute paths, drive letters, backslashes, NUL bytes, and any -/// separator other than the single `/` in a scoped name (`@scope/name`). +/// The alias is the install destination inside `node_modules` (renamed aside, +/// deleted and re-created), so on top of `is_safe_install_folder_name` it must +/// be a single path component, or two for a scoped name. pub(crate) fn alias_is_safe_install_target(alias: &[u8]) -> bool { - if alias.is_empty() || alias.len() >= MAX_PATH_BYTES || strings::contains_any(alias, b"\\:\0") { + if alias.len() >= MAX_PATH_BYTES || !crate::dependency::is_safe_install_folder_name(alias) { return false; } - let mut component_count = 0usize; - for component in strings::split(alias, b"/") { - component_count += 1; - if component.is_empty() || component == b"." || component == b".." { - return false; - } - } - + let component_count = strings::split(alias, b"/").count(); component_count == 1 || (component_count == 2 && alias[0] == b'@') } @@ -627,6 +623,7 @@ impl<'a> PackageInstaller<'a> { PostinstallOptimizer::NativeBinlink => { let target_cpu = manager.options.cpu; let target_os = manager.options.os; + let target_libc = manager.options.libc; if let Some(replacement_pkg_id) = PostinstallOptimizer::get_native_binlink_replacement_package_id( pkg_resolutions_lists[package_id as usize] @@ -634,6 +631,7 @@ impl<'a> PackageInstaller<'a> { pkg_metas, target_cpu, target_os, + target_libc, ) { let Some(target_tree_id) = find_native_binlink_target_tree( @@ -926,8 +924,7 @@ impl<'a> PackageInstaller<'a> { self.node_modules.path = context.path; self.current_tree_id = context.tree_id; - // Re-verify: a parent reinstall may have deleted a deferred entry. - const NEEDS_VERIFY: bool = true; + const NEEDS_VERIFY: bool = false; const IS_PENDING_PACKAGE_INSTALL: bool = true; self.install_package_with_name_and_resolution::( // This id might be different from the id used to enqueue the task. Important @@ -1061,6 +1058,28 @@ impl<'a> PackageInstaller<'a> { true } + /// Trees are visited in id order, so the parent's `replaced` is complete. + pub(crate) fn set_inside_replaced_folder(&mut self, tree_id: lockfile::tree::Id) { + let lockfile = self.lockfile(); + let tree = lockfile.buffers.trees.as_slice()[tree_id as usize]; + if tree.parent == lockfile::tree::INVALID_ID { + return; + } + debug_assert!(tree.parent < tree_id); + + let parent = &self.trees[tree.parent as usize]; + let inside = parent.inside_replaced_folder || { + let deps = lockfile.buffers.dependencies.as_slice(); + let string_buf = lockfile.buffers.string_bytes.as_slice(); + let folder = tree.folder_name(deps, string_buf); + parent + .replaced + .iter() + .any(|&dep_id| deps[dep_id as usize].name.slice(string_buf) == folder) + }; + self.trees[tree_id as usize].inside_replaced_folder = inside; + } + // `pub fn deinit` dropped. All owned fields (`pending_lifecycle_scripts: Vec`, // `completed_trees: Bitset`, `trees: Box<[TreeContext]>`, `tree_ids_to_trees_the_id_depends_on`, // `node_modules`, `trusted_dependencies_from_update_requests`) impl Drop. Borrowed fields @@ -1117,7 +1136,8 @@ impl<'a> PackageInstaller<'a> { // If a newly computed integrity hash is available (e.g. for a GitHub // tarball) and the lockfile doesn't already have one, persist it so - // the lockfile gets re-saved with the hash. + // the lockfile gets re-saved with the hash. Must happen before the + // callbacks below: a local tarball's cache entry is named after it. if data.integrity.tag.is_supported() { let pkg_metas = self.lockfile_mut().packages.items_meta_mut(); if !pkg_metas[package_id as usize].integrity.tag.is_supported() { @@ -1295,7 +1315,7 @@ impl<'a> PackageInstaller<'a> { if log_level != Options::LogLevel::Silent { bun_core::pretty_errorln!( "error: refusing to install dependency with unsafe name {}", - bstr::BStr::new(alias.slice(string_buf!())), + bun_core::fmt::escape_control_chars(alias.slice(string_buf!())), ); } self.summary.fail += 1; @@ -1307,22 +1327,11 @@ impl<'a> PackageInstaller<'a> { return; } - // `PackageInstall` stores both `destination_dir_subpath: &mut ZStr` - // and `destination_dir_subpath_buf: &mut [u8]` aliasing the same bytes. - // Derive BOTH from a single `*mut PathBuffer` - // so neither `&mut` invalidates the other under stacked-borrows. - let subpath_buf_ptr: *mut PathBuffer = &raw mut self.destination_dir_subpath_buf; - let destination_dir_subpath: &mut ZStr = { - let alias_slice = alias.slice(string_buf!()); - // SAFETY: `subpath_buf_ptr` is the unique borrow of the field; valid for - // the lifetime of this fn body. - let buf = unsafe { &mut *subpath_buf_ptr }; - buf[..alias_slice.len()].copy_from_slice(alias_slice); - buf[alias_slice.len()] = 0; - // SAFETY: buf[alias_slice.len()] == 0 written above; pointer derives from - // `subpath_buf_ptr` so it shares provenance with `destination_dir_subpath_buf` - // below. - unsafe { ZStr::from_raw_mut((*subpath_buf_ptr).as_mut_ptr(), alias_slice.len()) } + let destination_dir_subpath = { + let alias = alias.slice(string_buf!()); + self.destination_dir_subpath_buf[..alias.len()].copy_from_slice(alias); + self.destination_dir_subpath_buf[alias.len()] = 0; + ZStr::from_buf(&self.destination_dir_subpath_buf, alias.len()) }; let pkg_name_hash = self.pkg_name_hashes[package_id as usize]; @@ -1414,10 +1423,6 @@ impl<'a> PackageInstaller<'a> { }, cache_dir: Fd::INVALID, // assigned below destination_dir_subpath, - // SAFETY: `subpath_buf_ptr` = `&raw mut self.destination_dir_subpath_buf`; the - // field outlives `installer`. `destination_dir_subpath` above derives from the - // same raw pointer, so this `&mut` does not invalidate it under stacked-borrows. - destination_dir_subpath_buf: unsafe { (*subpath_buf_ptr).as_mut_slice() }, package_name: pkg_name, patch: patch_patch.map(|_| package_install::Patch { contents_hash: patch_contents_hash.unwrap(), @@ -1523,9 +1528,8 @@ impl<'a> PackageInstaller<'a> { } } resolution::Tag::LocalTarball => { - installer.cache_dir_subpath = package_manager::cached_tarball_folder_name( - self.manager_mut(), - *resolution.local_tarball(), + installer.cache_dir_subpath = package_manager::cached_local_tarball_folder_name( + &self.metas[package_id as usize].integrity, patch_contents_hash, ); installer.cache_dir = package_manager::get_cache_directory(self.manager_mut()); @@ -1568,16 +1572,31 @@ impl<'a> PackageInstaller<'a> { installer.cache_dir = Fd::cwd(); } else { let global_link_dir = package_manager::global_link_dir_path(self.manager_mut()); + let sep_len = (global_link_dir[global_link_dir.len() - 1] != SEP) as usize; + let len = global_link_dir.len() + sep_len + folder.len(); + // `folder` is the `link:` specifier as written in package.json. + if len >= self.folder_path_buf.len() { + if log_level != Options::LogLevel::Silent { + Output::err( + "ENAMETOOLONG", + "link path for package {} is too long", + (bstr::BStr::new(pkg_name.slice(string_buf!())),), + ); + } + self.summary.fail += 1; + self.increment_tree_install_count( + !IS_PENDING_PACKAGE_INSTALL, + self.current_tree_id, + log_level, + ); + return; + } let buf = self.folder_path_buf.as_mut_slice(); - let mut len = 0usize; - buf[len..len + global_link_dir.len()].copy_from_slice(global_link_dir); - len += global_link_dir.len(); - if global_link_dir[global_link_dir.len() - 1] != SEP { - buf[len] = SEP; - len += 1; + buf[..global_link_dir.len()].copy_from_slice(global_link_dir); + if sep_len != 0 { + buf[global_link_dir.len()] = SEP; } - buf[len..len + folder.len()].copy_from_slice(folder); - len += folder.len(); + buf[global_link_dir.len() + sep_len..len].copy_from_slice(folder); buf[len] = 0; // SAFETY: buf[len] == 0 written above installer.cache_dir_subpath = ZStr::from_buf(&self.folder_path_buf, len); @@ -1597,11 +1616,30 @@ impl<'a> PackageInstaller<'a> { } } - let needs_install = self.force_install - || self.skip_verify_installed_version_number - || !NEEDS_VERIFY + let verifying = + NEEDS_VERIFY && !self.force_install && !self.skip_verify_installed_version_number; + let inside_replaced_folder = + self.trees[self.current_tree_id as usize].inside_replaced_folder; + let needs_install = !verifying || remove_patch + || inside_replaced_folder || !installer.verify(resolution, &self.root_node_modules_folder); + self.summary.skipped += (!needs_install) as u32; + + // `install_from_link` (symlink/workspace) re-points one symlink; everything else + // replaces the folder. Child trees of a replaced folder inherit the flag. + if verifying + && needs_install + && !inside_replaced_folder + && !matches!( + resolution.tag, + resolution::Tag::Symlink | resolution::Tag::Workspace | resolution::Tag::Root + ) + { + self.trees[self.current_tree_id as usize] + .replaced + .push(dependency_id); + } if needs_install { if resolution.tag.can_enqueue_install_task() @@ -1830,14 +1868,16 @@ impl<'a> PackageInstaller<'a> { || (resolution.tag == resolution::Tag::Folder && !self.lockfile().is_workspace_tree_id(self.current_tree_id)) { - // This is a transitive folder dependency. It is installed with a single symlink to the target folder/file, - // and is not hoisted. + // This is a transitive folder dependency. It is installed as a directory of symlinks + // to the target's files (see `PackageInstall::install`), and is not hoisted. // // A transitive `Resolution::Folder` declared by a local `file:` package // is relative to the top-level dir (`Package::parse` normalized it), so // install it from `installer.cache_dir` (the cwd, set in the switch above). + // The same holds for a path from a root `overrides`/`resolutions` rule. if resolution.tag == resolution::Tag::Folder - && self.lockfile().is_folder_tree_id(self.current_tree_id) + && (self.lockfile().is_folder_tree_id(self.current_tree_id) + || self.lockfile().is_overridden_dependency(dependency_id)) { break 'result installer.install( self.skip_delete, @@ -1981,6 +2021,7 @@ impl<'a> PackageInstaller<'a> { self.lockfile().packages.items_meta(), self.manager().options.cpu, self.manager().options.os, + self.manager().options.libc, ) { if PackageManager::verbose_install() { @@ -2057,7 +2098,9 @@ impl<'a> PackageInstaller<'a> { "Blocked {} scripts for: {}@{}\n", count, bstr::BStr::new(alias.slice(string_buf!())), - resolution.fmt(string_buf!(), PathSep::Posix), + bun_core::fmt::for_terminal( + resolution.fmt(string_buf!(), PathSep::Posix) + ), ); } let entry = self @@ -2220,27 +2263,6 @@ impl<'a> PackageInstaller<'a> { } } } else { - // Same gate as the `needs_install` branch: a pending parent's - // `uninstall_before_install` would delete this verified package. - if !IS_PENDING_PACKAGE_INSTALL - && !Self::can_install_package_for_tree( - &self.completed_trees, - self.lockfile().buffers.trees.as_slice(), - self.current_tree_id, - ) - { - self.trees[self.current_tree_id as usize] - .pending_installs - .push(DependencyInstallContext { - dependency_id, - tree_id: self.current_tree_id, - path: self.node_modules.path.clone(), - }); - return; - } - - self.summary.skipped += 1; - if self.bins[package_id as usize].tag != bin::Tag::None { self.trees[self.current_tree_id as usize] .binaries @@ -2320,6 +2342,7 @@ impl<'a> PackageInstaller<'a> { self.lockfile().packages.items_meta(), self.manager().options.cpu, self.manager().options.os, + self.manager().options.libc, ) { if PackageManager::verbose_install() { diff --git a/src/install/PackageManager.rs b/src/install/PackageManager.rs index 242afc812fcb..7145759607f6 100644 --- a/src/install/PackageManager.rs +++ b/src/install/PackageManager.rs @@ -27,7 +27,7 @@ use bun_semver as Semver; use bun_sys::{self, Fd}; use bun_threading::{ThreadPool, UnboundedQueue, thread_pool}; use bun_transpiler as transpiler; -use bun_url::URL; +use bun_url::{OwnedURL, URL}; // `bun.spawn.process.WaiterThread` — the force-waiter-thread flag was moved // down into `bun_spawn::process` (MOVE_DOWN b0); install just flips it during @@ -80,6 +80,8 @@ pub mod populate_manifest_cache; pub mod process_dependency_list; #[path = "PackageManager/ProgressStrings.rs"] pub mod progress_strings; +#[path = "PackageManager/remove_stale_workspace_links.rs"] +pub(crate) mod remove_stale_workspace_links; #[path = "PackageManager/runTasks.rs"] pub mod run_tasks; #[path = "PackageManager/security_scanner.rs"] @@ -134,8 +136,8 @@ impl PackageManagerCommand { └ --quiet only output the tarball filename bun pm bin print the path to bin folder └ -g print the global path to bin folder - bun pm ls list the dependency tree according to the current lockfile - ├ --all list the entire dependency tree according to the current lockfile + bun pm ls list the tree of installed dependencies + ├ --all list the entire tree of installed dependencies └ --trusted list only trusted dependencies bun pm why \ show dependency tree explaining why a package is installed bun pm licenses list installed packages grouped by license @@ -178,12 +180,14 @@ Learn more about these at https://bun.com/docs/cli/pm. use crate::lockfile_real::package as Package; use crate::package_manager_task as Task; -use crate::resolvers::folder_resolver::{Entry as FolderResolutionEntry, FolderResolution}; +use crate::resolvers::folder_resolver::{ + Entry as FolderResolutionEntry, FolderResolution, Key as FolderResolutionKey, + Kind as FolderResolutionKind, +}; use bun_install::lockfile::{self, Lockfile}; use bun_install::{ - Dependency, DependencyID, NetworkTask, PackageID, PackageManifestMap, - PackageNameAndVersionHash, PackageNameHash, PatchTask, PreinstallState, TaskCallbackContext, - initialize_store, + DependencyID, NetworkTask, PackageID, PackageManifestMap, PackageNameAndVersionHash, + PackageNameHash, PatchTask, PreinstallState, TaskCallbackContext, initialize_store, }; // ────────────────────────────────────────────────────────────────────────── @@ -205,7 +209,8 @@ use directories::attempt_to_create_package_json_and_open; pub use directories::{ attempt_to_create_package_json, cached_git_folder_name, cached_git_folder_name_print, cached_git_folder_name_print_auto, cached_github_folder_name, cached_github_folder_name_print, - cached_github_folder_name_print_auto, cached_npm_package_folder_name, + cached_github_folder_name_print_auto, cached_local_tarball_folder_name, + cached_local_tarball_folder_name_print, cached_npm_package_folder_name, cached_npm_package_folder_name_print, cached_npm_package_folder_print_basename, cached_tarball_folder_name, cached_tarball_folder_name_print, compute_cache_dir_and_subpath, fetch_cache_directory_path, get_cache_directory, get_cache_directory_and_abs_path, @@ -269,8 +274,7 @@ type RepositoryMap = HashMap, 80 */> /// process. type AppendedTaskPackageMap = HashMap, 80 */>; -pub(crate) type FolderResolutionMap = - HashMap, 80 */>; +pub(crate) type FolderResolutionMap = HashMap; pub(crate) type NpmAliasMap = HashMap, 80 */>; @@ -281,7 +285,6 @@ pub type PatchTaskQueue = UnboundedQueue; pub type AsyncNetworkTaskQueue = UnboundedQueue; pub(crate) type SuccessFn = fn(&mut PackageManager, DependencyID, PackageID); -pub(crate) type FailFn = fn(&mut PackageManager, &Dependency, PackageID, Error); // Default to a maximum of 64 simultaneous HTTP requests for bun install if no proxy is specified // if a proxy IS specified, default to 64. We have different values because we might change this in the future. @@ -339,8 +342,6 @@ pub struct PackageManager { /// Only set in `bun pm` pub root_package_json_name_at_time_of_init: Box<[u8]>, - pub root_package_json_file: bun_sys::File, - /// The package id corresponding to the workspace the install is happening in. Could be root, or /// could be any of the workspaces. pub root_package_id: RootPackageId, @@ -449,6 +450,13 @@ pub struct PackageManager { // package.json cache entries that differ from disk; written by package_json_write_back::flush. pub(crate) edited_package_jsons: Vec, + // Set by package_json_write_back::flush when it rewrites a file; read by the install summary. + pub(crate) wrote_package_json: bool, + + // pnpm migration: what it moved into the cached root package.json. The file is only written along with the + // migrated lockfile (package_json_write_back::write_migrated_root); loads that are never saved leave it alone. + pub(crate) migrated_package_json_moves: Vec<&'static str>, + // bun add: catalog references decided per target and the root entries they need; see add_catalog.rs pub(crate) catalog_add: add_catalog::State, @@ -541,6 +549,14 @@ impl Subcommand { pub(crate) fn should_chdir_to_root(self) -> bool { !matches!(self, Self::Link) } + + /// `init` opens package.json read-write for these so a read-only file fails before any work. + pub(crate) fn writes_package_json(self, has_package_args: bool) -> bool { + matches!( + self, + Self::Add | Self::Remove | Self::Update | Self::Patch | Self::PatchCommit + ) || (matches!(self, Self::Install | Self::Link) && has_package_args) + } } /// The resolved outcome of `--filter` for one install: the importer ids whose dependencies get installed. @@ -586,10 +602,22 @@ pub struct PackageUpdateInfo { pub(crate) original_version_literal: Box<[u8]>, // set by the post-install write-back; the install summary still needs the entry pub(crate) written_back: bool, + /// Registered by `package_json_editor::record_catalog_originals`: the name's `catalog:` rows carry the move, so the install summary reports it through them. + pub(crate) catalog_entry: bool, pub(crate) original_version_string_buf: Box<[u8]>, pub(crate) original_version: Option, } +impl PackageUpdateInfo { + /// `version`'s tag strings live in `buf` (a lockfile string buffer that cleaning rebuilds), so the original keeps its own copy of them. + pub(crate) fn set_original_version(&mut self, version: Semver::Version, buf: &[u8]) { + // clone because the lockfile buffer may reallocate + let mut tag_buf = Vec::new(); + self.original_version = Some(version.clone_into(buf, &mut tag_buf)); + self.original_version_string_buf = tag_buf.into_boxed_slice(); + } +} + pub struct CatalogUpdateInfo { /// Catalog group name; empty for the default catalog. pub catalog_name: Box<[u8]>, @@ -623,7 +651,7 @@ pub enum TrackInstalledBin { Basename(Box<[u8]>), } -// MOVE_DOWN: data struct + accessors live in `bun_install_types::WakeHandler` +// MOVE_DOWN: the data struct lives in `bun_install_types::WakeHandler` // (single definition the resolver also stores). The `handler` second arg is // erased to `*mut c_void` there because that crate cannot name // `PackageManager`; `wake_raw()` casts it back at the call site. @@ -885,36 +913,14 @@ impl PackageManager { Ok(unsafe { &mut *ptr }) } - pub fn http_proxy(&self, url: &URL<'_>) -> Option> { - // `env_mut()` yields an unbounded `&'a Loader` (process-lifetime - // singleton), so the returned `URL<'_>` borrows for `'static`. - self.env_mut().get_http_proxy_for(url) + pub fn http_proxy(&self, url: &URL<'_>) -> Option { + self.env().get_http_proxy_for(url) } pub fn tls_reject_unauthorized(&self) -> bool { self.env().get_tls_reject_unauthorized() } - pub(crate) fn fail_root_resolution( - &mut self, - dependency: &Dependency, - dependency_id: DependencyID, - err: Error, - ) { - if let Some(ctx) = self.on_wake.context { - // SAFETY: `ctx` is the `WakeHandler::context` registered alongside - // this callback (a live `*mut Queue`); see `runtime::jsc_hooks`. - unsafe { - (self.on_wake.get_on_dependency_error())( - ctx.as_ptr(), - dependency, - dependency_id, - err.name(), - ); - } - } - } - /// Raw-pointer wake for concurrent task-thread callers (see /// `isolated_install::Installer::Task::callback`). Never materializes /// `&mut PackageManager`, so two task threads finishing simultaneously do @@ -931,12 +937,10 @@ impl PackageManager { // only form field pointers via `addr_of!`/`addr_of_mut!` (no whole-struct // borrow) and `wakeup()` is internally synchronized for cross-thread use. unsafe { - let on_wake = &*core::ptr::addr_of!((*this).on_wake); - if let Some(ctx) = on_wake.context { - // `WakeHandler.handler`'s second arg is the erased - // `*mut PackageManager` (`bun_install_types` cannot name this - // type); cast back to `*mut c_void` here. - (on_wake.get_handler())(ctx.as_ptr(), this.cast::()); + if let Some(wake) = (*core::ptr::addr_of!((*this).on_wake)).0 { + // `handler`'s second arg is the erased `*mut PackageManager` + // (`bun_install_types` cannot name this type). + (wake.handler)(wake.context.as_ptr(), this.cast::()); } (*core::ptr::addr_of_mut!((*this).event_loop)).wakeup(); } @@ -1377,6 +1381,7 @@ fn overlay_bunfig_install(install: &mut Api::BunInstall, bunfig: Api::BunInstall let Api::BunInstall { default_registry, scoped, + url_auth, lockfile_path, save_lockfile_path, cache_directory, @@ -1425,6 +1430,8 @@ fn overlay_bunfig_install(install: &mut Api::BunInstall, bunfig: Api::BunInstall } } + install.url_auth.extend(url_auth); + macro_rules! overlay { ($($field:ident),* $(,)?) => { $( if $field.is_some() { install.$field = $field; } )* @@ -1565,14 +1572,10 @@ pub fn init( let mut this_cwd: &[u8] = original_cwd; let mut created_package_json = false; let child_json: bun_sys::File = 'child: { - // if we are only doing `bun install` (no args), then we can open as read_only - // in all other cases we will need to write new data later. - // this is relevant because it allows us to succeed an install if package.json - // is readable but not writable - // - // probably wont matter as if package.json isn't writable, it's likely that - // the underlying directory and node_modules isn't either. - let need_write = subcommand != Subcommand::Install || cli.positionals.len() > 1; + // --dry-run and --no-save clear `Do::WRITE_PACKAGE_JSON` once the options load. + let need_write = subcommand.writes_package_json(cli.positionals.len() > 1) + && !cli.dry_run + && !cli.no_save; loop { let mut package_json_path_buf = PathBuffer::uninit(); @@ -1676,11 +1679,12 @@ pub fn init( parent_path_buf[parent_without_trailing_slash.len() + b"/package.json".len()] = 0; + // Finding the workspace root must not depend on its package.json being writable. let json_file = match bun_sys::File::openat( bun_sys::Fd::cwd(), &parent_path_buf [..parent_without_trailing_slash.len() + b"/package.json".len()], - bun_sys::O::RDWR | bun_sys::O::CLOEXEC, + bun_sys::O::RDONLY | bun_sys::O::CLOEXEC, 0, ) { Ok(f) => f, @@ -1692,16 +1696,20 @@ pub fn init( let json_stat_size = json_file.get_end_pos()?; let mut json_buf = vec![0u8; (json_stat_size + 64) as usize]; let json_len = json_file.pread_all(&mut json_buf, 0)?; + // The path as opened, not the fd's realpath, which may be on another drive + // (subst, junction) or outside the project (symlink) (#39357). + let json_path_len = + parent_without_trailing_slash.len() + b"/package.json".len(); // SAFETY: ROOT_PACKAGE_JSON_PATH_BUF is a process-global only touched on main // thread; `&raw mut` + explicit reborrow avoids the 2024 `static_mut_refs` deny. - let json_path = unsafe { - bun_sys::get_fd_path( - json_file.handle, - &mut *ROOT_PACKAGE_JSON_PATH_BUF.get(), - )? + let json_path: &[u8] = unsafe { + let root_buf = &mut *ROOT_PACKAGE_JSON_PATH_BUF.get(); + root_buf[..json_path_len] + .copy_from_slice(&parent_path_buf[..json_path_len]); + &root_buf[..json_path_len] }; let json_source = - bun_ast::Source::init_path_string(&*json_path, &json_buf[..json_len]); + bun_ast::Source::init_path_string(json_path, &json_buf[..json_len]); initialize_store(); // SAFETY: `ctx.log` is a borrow of the CLI's `Log`; valid for the // duration of `init()` (set by `Command::create()` before any install @@ -1804,10 +1812,6 @@ pub fn init( // process-lifetime (`set_top_level_dir` requires `'static`). fs.set_top_level_dir(fs.dirname_store().append(parent)?); let _ = child_json.close(); - #[cfg(windows)] - { - json_file.seek_to(0)?; - } workspace_name_hash = Some(Semver::string::Builder::string_hash(&entry.name)); break 'root_package_json_file json_file; @@ -1848,14 +1852,17 @@ pub fn init( // until now). The slice excludes the NUL — `top_level_dir` is `[]u8`. // PathBuffer is repr(transparent) over [u8; N], so the raw cast is sound. fs.set_top_level_dir(bun_core::ffi::slice(CWD_BUF.get().cast::(), tld.len())); - // bun_sys exposes the non-Z `get_fd_path`; - // append the NUL ourselves so the static `&ZStr` invariant holds. + // From `top_level_dir`, not the fd's realpath, so relative paths never cross drives (#39357). let root_buf = &mut *ROOT_PACKAGE_JSON_PATH_BUF.get(); - let p = bun_sys::get_fd_path(root_package_json_file.handle, root_buf)?; - let plen = p.len(); + let tld_no_slash = strings::without_trailing_slash(tld); + let plen = tld_no_slash.len() + SEP_PACKAGE_JSON.len(); + root_buf[..tld_no_slash.len()].copy_from_slice(tld_no_slash); + root_buf[tld_no_slash.len()..plen].copy_from_slice(SEP_PACKAGE_JSON); root_buf[plen] = 0; ROOT_PACKAGE_JSON_PATH.write(ZStr::from_raw(root_buf.as_ptr(), plen)); } + // From here on package.json is read (and, by the commands that do, written) by path. + let _ = root_package_json_file.close(); // Returns the resolver's BSSMap-owned // `*EntriesOption` slot. @@ -1910,24 +1917,27 @@ pub fn init( let npmrc_local = ZBox::from_bytes(b".npmrc"); let mut buf = PathBuffer::uninit(); + let parts = [b"./.npmrc" as &[u8]]; + // `None` (too long for `buf`) could not be opened either, so it counts as a missing file. + let join = resolve_path::join_abs_string_buf_z_checked::; + // npm's `userconfig` replaces both candidates below (actions/setup-node exports it). + let userconfig = [b"NPM_CONFIG_USERCONFIG" as &[u8], b"npm_config_userconfig"] + .into_iter() + .find_map(|key| env.get(key).filter(|path| !path.is_empty())); // npm reads `$HOME/.npmrc` and ignores XDG_CONFIG_HOME; keep // `$XDG_CONFIG_HOME/.npmrc` only when that file actually exists. let mut global_len: usize = 0; - if let Some(xdg_dir) = bun_core::env_var::XDG_CONFIG_HOME.get_not_empty() { - let p = - resolve_path::join_abs_string_buf_z::(xdg_dir, &mut buf, &parts); - if bun_sys::exists_z(p) { - global_len = p.len(); - } + if let Some(userconfig) = userconfig { + global_len = join(&original_cwd_clone, &mut buf, &[userconfig]).map_or(0, ZStr::len); + } else if let Some(xdg_dir) = bun_core::env_var::XDG_CONFIG_HOME.get_not_empty() { + global_len = (join(xdg_dir, &mut buf, &parts).filter(|p| bun_sys::exists_z(p))) + .map_or(0, ZStr::len); } - if global_len == 0 { + if global_len == 0 && userconfig.is_none() { if let Some(home_dir) = bun_core::env_var::HOME.get_not_empty() { - global_len = resolve_path::join_abs_string_buf_z::( - home_dir, &mut buf, &parts, - ) - .len(); + global_len = join(home_dir, &mut buf, &parts).map_or(0, ZStr::len); } } @@ -2048,7 +2058,6 @@ pub fn init( // zero-bit pattern is UB; allocate the real (empty) lockfile here directly. // `Lockfile::default()` ≡ `Lockfile::init_empty()`. wr!(lockfile, Box::new(Lockfile::default())); - wr!(root_package_json_file, root_package_json_file); // .progress wr!(event_loop, AnyEventLoop::init()); wr!( @@ -2122,6 +2131,8 @@ pub fn init( wr!(filtered_link_targets, None); wr!(pending_filtered_write, None); wr!(edited_package_jsons, Vec::new()); + wr!(wrote_package_json, false); + wr!(migrated_package_json_moves, Vec::new()); wr!(catalog_add, add_catalog::State::default()); wr!(patched_dependencies_to_remove, ArrayHashMap::default()); wr!(last_reported_slow_lifecycle_script_at, 0); @@ -2154,8 +2165,8 @@ pub fn init( { // make sure folder packages can find the root package without creating a new one // Posix-normalize the - // separators before hashing; `FolderResolution.hash` is always fed `/`-separated - // bytes by every resolver-side caller. On Windows `get_fd_path` yields `\`, so + // separators before hashing; the folder resolver always builds its `Key` from + // `/`-separated bytes. On Windows this path contains `\`, so // hashing the raw bytes would seed a key the resolver never looks up — copy into // a stack buffer and convert separators in place. // SAFETY: ROOT_PACKAGE_JSON_PATH set above on the main thread. @@ -2166,7 +2177,7 @@ pub fn init( resolve_path::dangerously_convert_path_to_posix_in_place::(normalized); // SAFETY: singleton fully initialized; main thread, no workers yet. unsafe { &mut *manager_ptr }.folders.put( - crate::resolvers::folder_resolver::hash(normalized), + FolderResolutionKey::new(FolderResolutionKind::Folder, normalized), FolderResolutionEntry { abs_path: Box::<[u8]>::from(&*normalized), resolution: FolderResolution::PackageId(0), @@ -2217,6 +2228,16 @@ pub fn init( } } + // `options.load` applies BUN_CONFIG_MAX_HTTP_REQUESTS on top of this default. + http::async_http::MAX_SIMULTANEOUS_REQUESTS.store( + if env.has_http_proxy() { + DEFAULT_MAX_SIMULTANEOUS_REQUESTS_FOR_BUN_INSTALL_FOR_PROXIES + } else { + DEFAULT_MAX_SIMULTANEOUS_REQUESTS_FOR_BUN_INSTALL + }, + Ordering::Relaxed, + ); + manager.options.load( // SAFETY: ctx.log is the process-lifetime CLI log set by // create_context_data(); single-threaded init region. @@ -2227,6 +2248,11 @@ pub fn init( subcommand, )?; + if let Some(network_concurrency) = cli_network_concurrency { + http::async_http::MAX_SIMULTANEOUS_REQUESTS + .store(usize::from(network_concurrency.max(1)), Ordering::Relaxed); + } + if let Some(config) = ctx.install.as_deref_mut() { if let Some(p) = config.public_hoist_pattern.take() { manager.options.public_hoist_pattern = Some(p); @@ -2275,22 +2301,6 @@ pub fn init( } } - http::async_http::MAX_SIMULTANEOUS_REQUESTS.store( - 'brk: { - if let Some(network_concurrency) = cli_network_concurrency { - break 'brk network_concurrency.max(1) as usize; - } - - // If any HTTP proxy is set, use a diferent limit - if env.has_http_proxy() { - break 'brk DEFAULT_MAX_SIMULTANEOUS_REQUESTS_FOR_BUN_INSTALL_FOR_PROXIES; - } - - DEFAULT_MAX_SIMULTANEOUS_REQUESTS_FOR_BUN_INSTALL - }, - Ordering::Relaxed, // .monotonic - ); - // `InitOpts.ca: Vec<*const c_void>` (erased `[*:0]const u8`). The HTTP // thread reads these asynchronously after `init` returns, so park the // owning `ZBox`es in `holder::CA` for process lifetime (never freed) @@ -2488,13 +2498,6 @@ fn init_with_runtime_once( // `Lockfile` holds `HashMap`/`Vec`/`NonNull` (zero-bit pattern is // UB), so allocate the real empty lockfile here directly instead of a zeroed placeholder. wr!(lockfile, Box::new(Lockfile::default())); - // `.root_package_json_file` is never read in the runtime - // path. Use the explicit invalid-fd sentinel rather than `mem::zeroed()` — - // on posix `Fd(0)` is stdin, not the invalid marker. - wr!( - root_package_json_file, - bun_sys::File::from_fd(Fd::invalid()) - ); // erased *mut () set by tier-6; `js_current()` resolves the per-thread JS // event loop via `bun_io::__bun_get_vm_ctx` (link-time, definer in bun_runtime). wr!(event_loop, AnyEventLoop::js_current()); @@ -2571,6 +2574,8 @@ fn init_with_runtime_once( wr!(filtered_link_targets, None); wr!(pending_filtered_write, None); wr!(edited_package_jsons, Vec::new()); + wr!(wrote_package_json, false); + wr!(migrated_package_json_moves, Vec::new()); wr!(catalog_add, add_catalog::State::default()); wr!(patched_dependencies_to_remove, ArrayHashMap::default()); wr!(last_reported_slow_lifecycle_script_at, 0); diff --git a/src/install/PackageManager/CommandLineArguments.rs b/src/install/PackageManager/CommandLineArguments.rs index be895af25bf1..6f65e9477432 100644 --- a/src/install/PackageManager/CommandLineArguments.rs +++ b/src/install/PackageManager/CommandLineArguments.rs @@ -32,6 +32,38 @@ fn pretty_help(text: &str) { Output::pretty(text); } +/// `--cpu` / `--os` / `--libc`, combined like the package.json arrays they filter. Unlike there +/// (`Negatable::apply` skips names this build does not know), an unknown name is a typo and fatal. +fn parse_platform_flag( + values: &[&[u8]], + unset: T, + what: &str, + valid_names: &str, +) -> T { + if values.is_empty() { + return unset; + } + let mut negatable = Npm::Negatable::::default(); + for &value in values { + match value { + b"*" => negatable.had_wildcard = true, + b"any" | b"none" => negatable.apply(value), + _ => { + let name = value.strip_prefix(b"!").unwrap_or(value); + if T::lookup_name(name).is_none() { + Output::err_generic( + "Invalid {}: '{}'. Valid values are: *, any, {}. Use !name to negate.", + (what, bstr::BStr::new(value), valid_names), + ); + Global::crash(); + } + negatable.apply(value); + } + } + } + negatable.combine() +} + type ParamType = clap::Param; // `bun_clap::concat_params!` is a const-fn slice concat over `Param`, so combined tables @@ -60,6 +92,7 @@ const PRODUCTION_PARAMS: &[ParamType] = &[ clap::param!("-P, --prod"), ]; +// `LINK_SHARED_PARAMS` swaps out the first two. const SHARED_TAIL_PARAMS: &[ParamType] = &[ clap::param!( "--no-save Don't update package.json or save a lockfile" @@ -89,7 +122,7 @@ const SHARED_TAIL_PARAMS: &[ParamType] = &[ "--no-verify Skip verifying integrity of newly downloaded packages" ), clap::param!( - "--ignore-scripts Skip lifecycle scripts in the project's package.json (dependency scripts are never run)" + "--ignore-scripts Skip lifecycle scripts for all packages, including the project's package.json and trusted dependencies" ), clap::param!( "--trust Add to trustedDependencies in the project's package.json and install the package(s)" @@ -104,7 +137,7 @@ const SHARED_TAIL_PARAMS: &[ParamType] = &[ "--concurrent-scripts Maximum number of concurrent jobs for lifecycle scripts (default: 2x CPU cores)" ), clap::param!( - "--network-concurrency Maximum number of concurrent network requests (default 48)" + "--network-concurrency Maximum number of concurrent network requests (default 64)" ), clap::param!("--save-text-lockfile Save a text-based lockfile"), clap::param!( @@ -125,12 +158,30 @@ const SHARED_TAIL_PARAMS: &[ParamType] = &[ clap::param!( "--os ... Override operating system for optional dependencies (e.g., linux, darwin, * for all)" ), + clap::param!( + "--libc ... Override libc for optional dependencies (e.g., glibc, musl, * for all)" + ), clap::param!("-h, --help Print this help menu"), ]; const SHARED_PARAMS: &[ParamType] = concat_params![SHARED_HEAD_PARAMS, PRODUCTION_PARAMS, SHARED_TAIL_PARAMS]; +// `parse` gives link and unlink the opposite default: they only save when --save is passed. +const LINK_SHARED_PARAMS: &[ParamType] = concat_params![ + SHARED_HEAD_PARAMS, + PRODUCTION_PARAMS, + &[ + clap::param!( + "--no-save Don't update package.json or save a lockfile (the default)" + ), + clap::param!( + "--save Update package.json and save a lockfile (false by default)" + ), + ], + SHARED_TAIL_PARAMS.split_at(2).1 +]; + pub(crate) static INSTALL_PARAMS: &[ParamType] = concat_params![ SHARED_PARAMS, &[ @@ -269,14 +320,14 @@ pub(crate) static REMOVE_PARAMS: &[ParamType] = concat_params![ ]; pub(crate) static LINK_PARAMS: &[ParamType] = concat_params![ - SHARED_PARAMS, + LINK_SHARED_PARAMS, &[clap::param!( " ... \"name\" install package as a link" ),] ]; pub(crate) static UNLINK_PARAMS: &[ParamType] = concat_params![ - SHARED_PARAMS, + LINK_SHARED_PARAMS, &[clap::param!( " ... \"name\" uninstall package as a link" ),] @@ -317,24 +368,49 @@ static OUTDATED_PARAMS: &[ParamType] = concat_params![ ] ]; -const AUDIT_PARAMS: &[ParamType] = &[ +static AUDIT_PARAMS: &[ParamType] = concat_params![ + SHARED_PARAMS, + &[ + clap::param!( + " ... Check installed packages for vulnerabilities" + ), + clap::param!("--json Output in JSON format"), + clap::param!( + "--audit-level Only print advisories with severity greater than or equal to \\ (low, moderate, high, critical)" + ), + clap::param!( + "--ignore ... Ignore advisories by GHSA or numeric advisory ID (repeatable)" + ), + clap::param!( + "-L, --latest Also apply fixes your declared ranges exclude, rewriting package.json" + ), + ] +]; + +const AUDIT_HELP_PARAMS: &[ParamType] = &[ clap::param!( - " ... Check installed packages for vulnerabilities" + "--audit-level Only print advisories with severity greater than or equal to \\ (low, moderate, high, critical)" ), - clap::param!("--json Output in JSON format"), clap::param!( - "--audit-level Only print advisories with severity greater than or equal to \\ (low, moderate, high, critical)" + "-p, --production Skip packages that are only needed by devDependencies (alias: --prod)" + ), + clap::param!( + "--omit ... Skip packages that are only needed by the given dependency types: dev, optional, or peer (repeatable)" ), clap::param!( "--ignore ... Ignore advisories by GHSA or numeric advisory ID (repeatable)" ), + clap::param!("--json Output in JSON format"), + clap::param!( + "--dry-run Show what bun audit fix would change without changing anything" + ), clap::param!( "-L, --latest Also apply fixes your declared ranges exclude, rewriting package.json" ), + clap::param!("--cwd Set a specific cwd"), + clap::param!("-h, --help Print this help menu"), ]; -static AUDIT_PARAMS_FULL: &[ParamType] = concat_params![SHARED_PARAMS, AUDIT_PARAMS]; - static INFO_PARAMS: &[ParamType] = concat_params![ SHARED_PARAMS, &[ @@ -402,6 +478,9 @@ static DEDUPE_PARAMS: &[ParamType] = concat_params![ clap::param!( "--check Exit with code 1 if the lockfile has duplicate versions that can be removed, without changing anything" ), + clap::param!( + "--why Also list each version's dependents and the ranges they asked for" + ), clap::param!(" ... "), ] ]; @@ -413,6 +492,9 @@ const DEDUPE_HELP_PARAMS: &[ParamType] = &[ clap::param!( "--dry-run Print the duplicate versions that would be removed without changing anything" ), + clap::param!( + "--why Also list each version's dependents and the ranges they asked for" + ), clap::param!("--lockfile-only Rewrite bun.lock without installing"), clap::param!( "--frozen-lockfile Fail instead of rewriting bun.lock when duplicate versions can be removed" @@ -428,6 +510,9 @@ const DEDUPE_HELP_PARAMS: &[ParamType] = &[ static PRUNE_PARAMS: &[ParamType] = concat_params![ SHARED_PARAMS, &[ + clap::param!( + "--check Exit with code 1 if node_modules has packages that can be removed, without deleting anything" + ), clap::param!( "-F, --filter ... Only prune the node_modules folders of the matching workspaces" ), @@ -442,6 +527,9 @@ const PRUNE_HELP_PARAMS: &[ParamType] = &[ clap::param!( "--omit ... Also remove packages that are only needed by the given dependency types" ), + clap::param!( + "--check Exit with code 1 if node_modules has packages that can be removed, without deleting anything" + ), clap::param!( "--dry-run Print what would be removed without deleting anything" ), @@ -451,6 +539,9 @@ const PRUNE_HELP_PARAMS: &[ParamType] = &[ clap::param!( "--cpu ... Prune for a different CPU architecture than the current one" ), + clap::param!( + "--libc ... Prune for a different libc than the current one" + ), clap::param!( "--linker Prune a node_modules installed with the given linker (one of \"isolated\" or \"hoisted\")" ), @@ -490,6 +581,7 @@ pub struct CommandLineArguments { pub(crate) no_save: bool, pub(crate) dry_run: bool, pub(crate) check: bool, + pub(crate) why: bool, pub(crate) force: bool, pub(crate) no_cache: bool, pub log_level: Options::LogLevel, @@ -554,9 +646,10 @@ pub struct CommandLineArguments { pub audit_level: Option, pub audit_ignore_list: &'static [&'static [u8]], - // CPU and OS overrides for optional dependencies + // CPU, OS and libc overrides for optional dependencies pub(crate) cpu: Npm::Architecture, pub(crate) os: Npm::OperatingSystem, + pub(crate) libc: Npm::Libc, } impl Default for CommandLineArguments { @@ -580,6 +673,7 @@ impl Default for CommandLineArguments { no_save: false, dry_run: false, check: false, + why: false, force: false, no_cache: false, log_level: Options::LogLevel::default(), @@ -642,6 +736,7 @@ impl Default for CommandLineArguments { cpu: Npm::Architecture::CURRENT, os: Npm::OperatingSystem::CURRENT, + libc: Npm::Libc::CURRENT, } } } @@ -905,9 +1000,12 @@ Full documentation is available at https://bun.com/docs/cli/remove. Directory should contain a package.json. bun link - Add a previously-registered linkable package as a dependency of the current project. + Link a previously-registered linkable package into the current project's node_modules. bun link \ + Also add it to the current project's package.json as a link: dependency. + bun link --save \ + Full documentation is available at https://bun.com/docs/cli/link. "; pretty_help(intro_text); @@ -1052,7 +1150,7 @@ Full documentation is available at https://bun.com/docs/install/audithttps://bun.com/docs/cli/why. Show what would be removed without changing anything bun dedupe --dry-run + Show which dependents wanted each version + bun dedupe --dry-run --why + Rewrite bun.lock without installing bun dedupe --lockfile-only @@ -1158,6 +1259,9 @@ Full documentation is available at https://bun.com/docs/pm/cli/dedupeShow what would be removed without deleting anything bun prune --dry-run + Only report what would be removed; exit code 1 if there is anything (for CI) + bun prune --check + Only prune what the app workspace no longer needs bun prune --production --filter app @@ -1191,10 +1295,7 @@ Full documentation is available at https://bun.com/docs/pm/cli/prune Subcommand::Why => WHY_PARAMS, Subcommand::Dedupe => DEDUPE_PARAMS, Subcommand::Prune => PRUNE_PARAMS, - - // TODO: we will probably want to do this for other *_params. this way extra params - // are not included in the help text - Subcommand::Audit => AUDIT_PARAMS_FULL, + Subcommand::Audit => AUDIT_PARAMS, Subcommand::Info => INFO_PARAMS, }; @@ -1359,10 +1460,13 @@ Full documentation is available at https://bun.com/docs/pm/cli/prune // cli.json_output = args.flag(b"--json"); } - if subcommand == Subcommand::Dedupe && args.flag(b"--check") { + if matches!(subcommand, Subcommand::Dedupe | Subcommand::Prune) && args.flag(b"--check") { cli.check = true; cli.dry_run = true; } + if subcommand == Subcommand::Dedupe { + cli.why = args.flag(b"--why"); + } if matches!( subcommand, @@ -1470,59 +1574,19 @@ Full documentation is available at https://bun.com/docs/pm/cli/prune cli.config = Some(opt); } - // Parse multiple --cpu flags and combine them using Negatable - let cpu_values = args.options(b"--cpu"); - if !cpu_values.is_empty() { - let mut cpu_negatable = Npm::Architecture::NONE.negatable(); - for cpu_str in cpu_values { - // apply() already handles "any" as wildcard and negation with ! - cpu_negatable.apply(cpu_str); - - // Support * as an alias for "any" - if *cpu_str == *b"*" { - cpu_negatable.had_wildcard = true; - cpu_negatable.had_unrecognized_values = false; - } else if cpu_negatable.had_unrecognized_values - && *cpu_str != *b"any" - && *cpu_str != *b"none" - { - // Only error for truly unrecognized values (not "any" or "none") - Output::err_generic( - "Invalid CPU architecture: '{}'. Valid values are: *, any, arm, arm64, ia32, mips, mipsel, ppc, ppc64, s390, s390x, x32, x64. Use !name to negate.", - (bstr::BStr::new(cpu_str),), - ); - Global::crash(); - } - } - cli.cpu = cpu_negatable.combine(); - } - - // Parse multiple --os flags and combine them using Negatable - let os_values = args.options(b"--os"); - if !os_values.is_empty() { - let mut os_negatable = Npm::OperatingSystem::NONE.negatable(); - for os_str in os_values { - // apply() already handles "any" as wildcard and negation with ! - os_negatable.apply(os_str); - - // Support * as an alias for "any" - if *os_str == *b"*" { - os_negatable.had_wildcard = true; - os_negatable.had_unrecognized_values = false; - } else if os_negatable.had_unrecognized_values - && *os_str != *b"any" - && *os_str != *b"none" - { - // Only error for truly unrecognized values (not "any" or "none") - Output::err_generic( - "Invalid operating system: '{}'. Valid values are: *, any, aix, darwin, freebsd, linux, openbsd, sunos, win32, android. Use !name to negate.", - (bstr::BStr::new(os_str),), - ); - Global::crash(); - } - } - cli.os = os_negatable.combine(); - } + cli.cpu = parse_platform_flag( + args.options(b"--cpu"), + cli.cpu, + "CPU architecture", + "arm, arm64, ia32, mips, mipsel, ppc, ppc64, s390, s390x, x32, x64", + ); + cli.os = parse_platform_flag( + args.options(b"--os"), + cli.os, + "operating system", + "aix, darwin, freebsd, linux, openbsd, sunos, win32, android", + ); + cli.libc = parse_platform_flag(args.options(b"--libc"), cli.libc, "libc", "glibc, musl"); if matches!(subcommand, Subcommand::Add | Subcommand::Install) { cli.dependency_group = if args.flag(b"--development") || args.flag(b"--dev") { @@ -1550,28 +1614,28 @@ Full documentation is available at https://bun.com/docs/pm/cli/prune let mut buf = PathBuffer::uninit(); let mut buf2 = PathBuffer::uninit(); - let final_path: &mut bun_core::ZStr = if !cwd_.is_empty() && cwd_[0] == b'.' { + let final_path: Option<&bun_core::ZStr> = if !cwd_.is_empty() && cwd_[0] == b'.' { let cwd_len = bun_sys::getcwd(&mut buf[..])?; let cwd = &buf[..cwd_len]; let parts: [&[u8]; 1] = [cwd_]; - let len = Path::resolve_path::join_abs_string_buf::( + Path::resolve_path::join_abs_string_buf_z_checked::( cwd, &mut buf2[..], &parts, ) - .len(); - buf2[len] = 0; - bun_core::ZStr::from_buf_mut(&mut buf2[..], len) - } else { + } else if cwd_.len() < buf.len() { buf[..cwd_.len()].copy_from_slice(cwd_); buf[cwd_.len()] = 0; - bun_core::ZStr::from_buf_mut(&mut buf[..], cwd_.len()) + Some(bun_core::ZStr::from_buf(&buf[..], cwd_.len())) + } else { + None }; - if let Err(err) = bun_sys::chdir(final_path) { + let too_long = bun_sys::Error::from_code(bun_sys::E::ENAMETOOLONG, bun_sys::Tag::chdir); + if let Err(err) = final_path.map_or(Err(too_long), bun_sys::chdir) { Output::err_generic( "failed to change directory to \"{}\": {}\n", ( - bstr::BStr::new(final_path.as_bytes()), + bstr::BStr::new(final_path.map_or(cwd_, |p| p.as_bytes())), bstr::BStr::new(err.name()), ), ); diff --git a/src/install/PackageManager/PackageJSONEditor.rs b/src/install/PackageManager/PackageJSONEditor.rs index 00a19a9c55f3..543899e7273f 100644 --- a/src/install/PackageManager/PackageJSONEditor.rs +++ b/src/install/PackageManager/PackageJSONEditor.rs @@ -437,6 +437,11 @@ fn edit_update_aliases_of_requests( ) } +/// The tag of a package.json literal as `bun install` reads it (leading whitespace ignored). +fn literal_tag(literal: &[u8]) -> dependency::Tag { + dependency::Tag::infer(dependency::trim_literal(literal)) +} + fn edit_update_entries( lockfile: &crate::Lockfile, arena: &bun_alloc::Arena, @@ -480,7 +485,7 @@ fn edit_update_entries( let version_literal = value .as_utf8_string_literal() .unwrap_or_else(|| bun_core::out_of_memory()); - let tag = dependency::Tag::infer(version_literal); + let tag = literal_tag(version_literal); // npm ranges only (and dist-tags with --latest); `catalog:` is handled by edit_catalogs_*. if tag != dependency::Tag::Npm @@ -507,9 +512,7 @@ fn edit_update_entries( *entry.value_ptr = PackageUpdateInfo { original_version_literal: version_literal_owned, - written_back: false, - original_version_string_buf: Box::default(), - original_version: None, + ..Default::default() }; if update_to_latest { @@ -560,7 +563,7 @@ fn edit_update_entries( let value_literal = value .as_utf8_string_literal() .unwrap_or_else(|| bun_core::out_of_memory()); - if dependency::Tag::infer(value_literal) == dependency::Tag::Catalog { + if literal_tag(value_literal) == dependency::Tag::Catalog { continue; } @@ -717,7 +720,7 @@ pub(crate) fn edit_catalogs_before_update( let version_literal = value .as_utf8_string_literal() .unwrap_or_else(|| bun_core::out_of_memory()); - let tag = dependency::Tag::infer(version_literal); + let tag = literal_tag(version_literal); // same tag rule as direct dependencies if tag != dependency::Tag::Npm && (tag != dependency::Tag::DistTag || !update_to_latest) @@ -751,6 +754,56 @@ pub(crate) fn edit_catalogs_before_update( Ok(!manager.updating_catalogs.is_empty()) } +/// Runs on the loaded lockfile, before the differ: every `catalog:` row of an entry recorded by `edit_catalogs_before_update` registers its name in `updating_packages` with the row's locked version as the original, the way the cwd's own dependency lists register theirs, so the install summary prints the entry's move as an update row; a name those lists already registered keeps their original. +pub(crate) fn record_catalog_originals( + manager: &mut PackageManager, +) -> Result<(), bun_alloc::AllocError> { + let infos: &[CatalogUpdateInfo] = &manager.updating_catalogs; + if infos.is_empty() { + return Ok(()); + } + let by_name = CatalogInfoIndex::init(infos)?; + let lockfile: &Lockfile = &manager.lockfile; + let updating_packages = &mut manager.updating_packages; + let string_buf = lockfile.buffers.string_bytes.as_slice(); + let package_resolutions = lockfile.packages.items_resolution(); + + let dependencies = lockfile.buffers.dependencies.iter(); + for (dep, &package_id) in dependencies.zip(lockfile.buffers.resolutions.iter()) { + if dep.version.tag != dependency::Tag::Catalog { + continue; + } + let resolution = package_resolutions.get(package_id as usize); + let Some(resolution) = resolution.filter(|r| r.tag == resolution::Tag::Npm) else { + continue; + }; + let dep_name = dep.name.slice(string_buf); + let catalog_name = dep.version.catalog().slice(string_buf); + let Some(info) = by_name + .candidates(dep_name) + .and_then(|candidates| CatalogInfoIndex::pick(candidates, infos, catalog_name)) + .map(|i| &infos[i]) + else { + continue; + }; + let entry = updating_packages.get_or_put(dep_name)?; + if entry.found_existing { + continue; + } + *entry.value_ptr = PackageUpdateInfo { + original_version_literal: info.original_version_literal.clone(), + // The entry is written by `edit_catalogs_after_update`; `edit_update_entries` has nothing of it to write into the cwd's dependency lists. + written_back: true, + catalog_entry: true, + ..Default::default() + }; + entry + .value_ptr + .set_original_version(resolution.npm().version, string_buf); + } + Ok(()) +} + /// Writes each recorded catalog entry's resolved literal (unresolved ones are restored) into the root AST; returns `changed`. pub(crate) fn edit_catalogs_after_update( manager: &mut PackageManager, @@ -996,7 +1049,7 @@ pub(crate) fn edit( == Subcommand::Update && value.expr.as_utf8_string_literal().is_some_and( |version_literal| { - dependency::Tag::infer(version_literal) + literal_tag(version_literal) == dependency::Tag::Catalog }, ); @@ -1018,7 +1071,7 @@ pub(crate) fn edit( else { break 'add_packages_to_update; }; - let tag = dependency::Tag::infer(version_literal); + let tag = literal_tag(version_literal); if tag != dependency::Tag::Npm && tag != dependency::Tag::DistTag @@ -1041,9 +1094,7 @@ pub(crate) fn edit( *entry.value_ptr = PackageUpdateInfo { original_version_literal: version_literal_owned, - written_back: false, - original_version_string_buf: Box::default(), - original_version: None, + ..Default::default() }; } } @@ -1194,10 +1245,18 @@ pub(crate) fn edit( bun_ast::Loc::EMPTY, )); + // Read by the `workspace:` arm of the write-back below; every other arm overwrites it. + let declared: &[u8] = match new_dependencies[k] + .value + .as_ref() + .and_then(Expr::as_utf8_string_literal) + { + Some(literal) => arena_dup(arena, literal), + None => b"", + }; new_dependencies[k].value = Some(Expr::allocate( arena, - // we set it later - E::EString::init(b""), + E::EString::init(declared), bun_ast::Loc::EMPTY, )); @@ -1332,9 +1391,9 @@ pub(crate) fn edit( // derived from a `StoreRef` to the same `E::EString` is live inside this loop body, // so this is the sole mutable borrow. let e_string = unsafe { &mut *e_string }; - // `bun update ` keeps a `catalog:` reference; `bun add` still replaces it. + // `bun update ` only moves registry entries, like `edit_update_entries`; `bun add` still replaces any entry. if manager.subcommand == Subcommand::Update - && dependency::Tag::infer(e_string.data.slice()) == dependency::Tag::Catalog + && !literal_tag(e_string.data.slice()).is_npm() { continue; } @@ -1447,7 +1506,13 @@ pub(crate) fn edit( arena_dup(arena, installed) } - resolution::Tag::Workspace => b"workspace:*", + // A range that linked a workspace member has nothing to move to; `workspace:*` is what `bun add` writes. + resolution::Tag::Workspace if manager.subcommand == Subcommand::Update => continue, + // Not the bound row: an unchanged resolution keeps the old lockfile row and its previous literal. + resolution::Tag::Workspace => match literal_tag(e_string.data.slice()) { + dependency::Tag::Workspace => e_string.data.slice(), + _ => b"workspace:*", + }, _ => arena_dup(arena, request.version.literal.slice(request.version_buf())), }; if e_string.data.slice() != new_literal { diff --git a/src/install/PackageManager/PackageManagerDirectories.rs b/src/install/PackageManager/PackageManagerDirectories.rs index 4861484c4b21..eaa0caa12d86 100644 --- a/src/install/PackageManager/PackageManagerDirectories.rs +++ b/src/install/PackageManager/PackageManagerDirectories.rs @@ -11,7 +11,7 @@ use bun_core::{Global, Output, ZBox, env_var, fmt as bun_fmt}; use bun_dotenv::Loader as DotEnvLoader; use bun_install::lockfile::{Format as LockfileFormat, LoadResult, Lockfile}; use bun_install::resolution::Tag as ResolutionTag; -use bun_install::{PackageID, Resolution}; +use bun_install::{Integrity, PackageID, Resolution}; use bun_paths::{self as path, AbsPath, PathBuffer, SEP}; use bun_semver::{self as Semver, String as SemverString}; #[cfg(windows)] @@ -242,6 +242,8 @@ fn get_temporary_directory_run(manager: &mut PackageManager) -> TemporaryDirecto match sys::renameat_z(tempdir.fd(), tmpname, cache_directory.fd(), tmpname) { Ok(()) => {} Err(err) => { + // The rename failed, so the probe is still sitting in `tempdir`. + let _ = tempdir.delete_file_z(tmpname); if !tried_dot_tmp { tried_dot_tmp = true; tempdir = match cache_directory.make_open_path(b".tmp", Default::default()) { @@ -306,9 +308,20 @@ fn get_temporary_directory_run(manager: &mut PackageManager) -> TemporaryDirecto } }; + // `name` must be the directory `handle` is: the node-gyp shim is written via `handle`, put on PATH via `name`. + let name: &'static [u8] = if tried_dot_tmp { + let joined = path::resolve_path::join::(&[ + manager.cache_directory_path.as_bytes(), + b".tmp", + ]); + bun_core::handle_oom(FileSystem::instance().dirname_store().append(joined)) + } else { + temp_dir_name + }; + TemporaryDirectory { handle: tempdir, - name: temp_dir_name, + name, #[cfg(windows)] path: ZBox::from_bytes(temp_dir_path.as_bytes()), } @@ -332,11 +345,14 @@ unsafe fn ensure_cache_directory(this: *mut PackageManager) -> Dir { // encapsulates the BackRef deref + singleton-liveness invariant. let env = unsafe { &*this }.env_mut(); // SAFETY: shared read of `options`; disjoint from `cache_directory_path`. - let cache_dir = fetch_cache_directory_path(env, Some(unsafe { &(*this).options })); - // SAFETY: see fn safety contract. - unsafe { (*this).cache_directory_path = ZBox::from_bytes(&cache_dir.path) }; + let opened = fetch_cache_directory_path(env, Some(unsafe { &(*this).options })) + .and_then(|cache_dir| { + // SAFETY: see fn safety contract. + unsafe { (*this).cache_directory_path = ZBox::from_bytes(&cache_dir.path) }; + Dir::cwd().make_open_path(&cache_dir.path, Default::default()) + }); - match Dir::cwd().make_open_path(&cache_dir.path, Default::default()) { + match opened { Ok(d) => return d, Err(_) => { // SAFETY: narrow `&mut enable` projection; disjoint from @@ -375,46 +391,34 @@ pub struct CacheDir { pub path: Vec, } -pub fn fetch_cache_directory_path(env: &mut DotEnvLoader, options: Option<&Options>) -> CacheDir { - if let Some(dir) = env.get(b"BUN_INSTALL_CACHE_DIR") { - return CacheDir { - path: FileSystem::instance().abs(&[dir]).to_vec(), - }; - } - - if let Some(opts) = options { - if !opts.cache_directory.is_empty() { - return CacheDir { - path: FileSystem::instance().abs(&[opts.cache_directory]).to_vec(), - }; - } - } - - if let Some(dir) = env.get(b"BUN_INSTALL") { - let parts: [&[u8]; 3] = [dir, b"install/", b"cache/"]; - return CacheDir { - path: FileSystem::instance().abs(&parts).to_vec(), - }; - } - - if let Some(dir) = env_var::XDG_CACHE_HOME.get() { - let parts: [&[u8]; 4] = [dir, b".bun/", b"install/", b"cache/"]; - return CacheDir { - path: FileSystem::instance().abs(&parts).to_vec(), - }; - } - - if let Some(dir) = env_var::HOME.get() { - let parts: [&[u8]; 4] = [dir, b".bun/", b"install/", b"cache/"]; - return CacheDir { - path: FileSystem::instance().abs(&parts).to_vec(), - }; - } +/// `ENAMETOOLONG` when the configured directory does not fit a `PathBuffer`. +pub fn fetch_cache_directory_path( + env: &mut DotEnvLoader, + options: Option<&Options>, +) -> sys::Maybe { + let parts: &[&[u8]] = if let Some(dir) = env.get(b"BUN_INSTALL_CACHE_DIR") { + &[dir] + } else if let Some(dir) = options + .map(|opts| opts.cache_directory) + .filter(|dir| !dir.is_empty()) + { + &[dir] + } else if let Some(dir) = env.get(b"BUN_INSTALL") { + &[dir, b"install/", b"cache/"] + } else if let Some(dir) = env_var::XDG_CACHE_HOME + .get() + .or_else(|| env_var::HOME.get()) + { + &[dir, b".bun/", b"install/", b"cache/"] + } else { + &[b"node_modules/.bun-cache"] + }; - let fallback_parts: [&[u8]; 1] = [b"node_modules/.bun-cache"]; - CacheDir { - path: FileSystem::instance().abs(&fallback_parts).to_vec(), - } + let mut buf = path::path_buffer_pool::get(); + let Some(abs) = FileSystem::instance().abs_buf_checked(parts, &mut buf[..]) else { + return Err(sys::Error::from_code(sys::E::ENAMETOOLONG, sys::Tag::open).with_path(parts[0])); + }; + Ok(CacheDir { path: abs.to_vec() }) } // ─────────────────────── cached folder name printers ────────────────────────── @@ -487,6 +491,14 @@ impl<'a> ByteCursor<'a> { self.put(bun_fmt::u64_hex_var_lower(&mut tmp, n)); } + /// Two lower-hex digits per byte. + #[inline(always)] + fn put_hex_bytes(&mut self, bytes: &[u8]) { + let end = self.at + bytes.len() * 2; + bun_fmt::bytes_to_hex_lower(bytes, &mut self.buf[self.at..end]); + self.at = end; + } + /// `@@@{d}` when set. #[inline(always)] fn put_cache_version(&mut self, v: Option) { @@ -725,6 +737,7 @@ pub fn cached_npm_package_folder_print_basename<'a>( w.finish_z() } +/// `@T@@@@1`; `file:` tarballs use [`cached_local_tarball_folder_name_print`]. pub fn cached_tarball_folder_name_print<'a>( buf: &'a mut [u8], url: &[u8], @@ -750,6 +763,35 @@ pub fn cached_tarball_folder_name( ) } +/// `@T@sha512-@@@1`: a `file:` tarball's path names different bytes +/// in every project sharing the cache, so it is cached under the integrity bun.lock pins instead. +/// Empty while that integrity is unknown, which callers treat as a cache miss. +pub fn cached_local_tarball_folder_name_print<'a>( + buf: &'a mut [u8], + integrity: &Integrity, + patch_hash: Option, +) -> &'a ZStr { + let Some(algorithm) = integrity.tag.name() else { + return ZStr::EMPTY; + }; + let digest = integrity.slice(); + let mut w = ByteCursor::new(buf); + w.put(b"@T@"); + w.put(algorithm.as_bytes()); + w.put_byte(b'-'); + w.put_hex_bytes(&digest[..digest.len().min(16)]); + w.put_cache_version(Some(CacheVersion::CURRENT)); + w.put_patch_hash(patch_hash); + w.finish_z() +} + +pub fn cached_local_tarball_folder_name( + integrity: &Integrity, + patch_hash: Option, +) -> &'static ZStr { + cached_local_tarball_folder_name_print(cached_package_folder_name_buf(), integrity, patch_hash) +} + pub fn is_folder_in_cache(this: &mut PackageManager, folder_path: &ZStr) -> bool { sys::directory_exists_at(get_cache_directory(this), folder_path).unwrap_or(false) } @@ -947,6 +989,7 @@ pub fn compute_cache_dir_and_subpath<'a>( manager: &mut PackageManager, pkg_name: &[u8], resolution: &Resolution, + integrity: &Integrity, folder_path_buf: &'a mut PathBuffer, patch_hash: Option, ) -> CacheDirAndSubpath<'a> { @@ -986,8 +1029,20 @@ pub fn compute_cache_dir_and_subpath<'a>( cache_dir = Fd::cwd(); } ResolutionTag::LocalTarball => { - let tarball = *resolution.local_tarball(); - cache_dir_subpath = cached_tarball_folder_name(manager, tarball, patch_hash); + cache_dir_subpath = cached_local_tarball_folder_name(integrity, patch_hash); + if cache_dir_subpath.is_empty() { + Output::err_generic( + "the lockfile does not record an integrity for {}@{}, run bun install first", + ( + bun_fmt::s(name), + resolution.fmt( + manager.lockfile.buffers.string_bytes.as_slice(), + bun_fmt::PathSep::Posix, + ), + ), + ); + Global::exit(1); + } cache_dir = get_cache_directory(manager); } ResolutionTag::RemoteTarball => { diff --git a/src/install/PackageManager/PackageManagerEnqueue.rs b/src/install/PackageManager/PackageManagerEnqueue.rs index 72aa9715ba81..a8b86dbf4709 100644 --- a/src/install/PackageManager/PackageManagerEnqueue.rs +++ b/src/install/PackageManager/PackageManagerEnqueue.rs @@ -20,7 +20,7 @@ use crate::lockfile::PackageIndexEntry; use crate::lockfile::package::Package; use crate::lockfile_real as Lockfile; use crate::package_manager_real::{ - self, FailFn, PackageManager, SuccessFn, TaskCallbackList, determine_preinstall_state, + self, PackageManager, SuccessFn, TaskCallbackList, determine_preinstall_state, get_cache_directory, get_preinstall_state, get_temporary_directory, run_tasks, set_preinstall_state, }; @@ -50,15 +50,6 @@ fn verbose_install() -> bool { // `PatchTask.callback` discriminant — routed to the real // `patch_install::Callback` enum (CalcHash / Apply). -// `SuccessFn` / `FailFn` are bare `fn(&mut PackageManager, ...)` pointers; the -// real bodies are inherent methods, so reference them via the type path. -#[allow(non_upper_case_globals)] -const assign_resolution: SuccessFn = PackageManager::assign_resolution; -#[allow(non_upper_case_globals)] -const assign_root_resolution: SuccessFn = PackageManager::assign_root_resolution; -#[allow(non_upper_case_globals)] -const fail_root_resolution: FailFn = PackageManager::fail_root_resolution; - // The `use package_manager_real::PackageManager` // above already pulls the `declare_scope!`-generated `static PackageManager: ScopedLogger` // (value namespace) alongside the struct (type namespace), so re-declaring it here @@ -85,8 +76,7 @@ pub fn enqueue_dependency_with_main( dependency, resolution, install_peer, - assign_resolution, - None, + PackageManager::assign_resolution, false, ) } @@ -108,37 +98,7 @@ pub fn enqueue_dependency_list( let dependency = this.lockfile.buffers.dependencies[i as usize].clone(); let resolution = this.lockfile.buffers.resolutions[i as usize]; if let Err(err) = enqueue_dependency_with_main(this, i, &dependency, resolution, false) { - let path_sep = match dependency.version.tag { - dependency::version::Tag::Folder => bun_fmt::PathSep::Auto, - _ => bun_fmt::PathSep::Any, - }; - // `format_args!` borrows temporaries — bind the - // formatter first so it outlives the macro expansion. - let realname = dependency.realname(); - let path_fmt = bun_fmt::fmt_path_u8( - this.lockfile.str(&realname), - bun_fmt::PathFormatOptions { - path_sep, - escape_backslashes: false, - }, - ); - let log = this.log_mut(); - if dependency.behavior.is_optional() || dependency.behavior.is_peer() { - log.add_warning_with_note( - None, - bun_ast::Loc::default(), - err.name().as_bytes(), - format_args!("error occurred while resolving {}", path_fmt), - ); - } else { - log.add_zig_error_with_note( - err.name(), - format_args!("error occurred while resolving {}", path_fmt), - ); - } - - i += 1; - continue; + add_dependency_error(this, &dependency, err); } i += 1; } @@ -146,6 +106,42 @@ pub fn enqueue_dependency_list( this.drain_dependency_list(); } +/// Logged to `this.log`; the install fails later when `has_errors()` is checked. +#[cold] +#[inline(never)] +pub(crate) fn add_dependency_error( + this: &mut PackageManager, + dependency: &Dependency, + err: crate::Error, +) { + let path_sep = match dependency.version.tag { + dependency::version::Tag::Folder => bun_fmt::PathSep::Auto, + _ => bun_fmt::PathSep::Any, + }; + let realname = dependency.realname(); + let path_fmt = bun_fmt::EscapeControlChars(bun_fmt::fmt_path_u8( + this.lockfile.str(&realname), + bun_fmt::PathFormatOptions { + path_sep, + escape_backslashes: false, + }, + )); + let log = this.log_mut(); + if dependency.behavior.is_optional() || dependency.behavior.is_peer() { + log.add_warning_with_note( + None, + bun_ast::Loc::default(), + err.name().as_bytes(), + format_args!("error occurred while resolving {}", path_fmt), + ); + } else { + log.add_zig_error_with_note( + err.name(), + format_args!("error occurred while resolving {}", path_fmt), + ); + } +} + pub fn enqueue_tarball_for_download( this: &mut PackageManager, dependency_id: DependencyID, @@ -462,8 +458,7 @@ pub fn enqueue_dependency_to_root( &dependency, invalid_package_id, false, - assign_root_resolution, - Some(fail_root_resolution), + PackageManager::assign_root_resolution, true, ) { return DependencyToEnqueue::Failure(err); @@ -645,12 +640,8 @@ pub fn enqueue_dependency_with_main_and_success_fn( resolution: PackageID, install_peer: bool, success_fn: SuccessFn, - fail_fn: Option, - // The two `SuccessFn` candidates - // (`assign_resolution` / `assign_root_resolution`) have byte-identical - // bodies in release builds, so Apple ld64 (which ignores `.llvm_addrsig`) - // folds them and a runtime fn-pointer address comparison is unsound. Thread - // an explicit flag instead. + // `enqueue_dependency_to_root` (auto-install): resolution errors go to `this.log` instead of propagating. + // A flag rather than a `success_fn` address comparison, which linker folding would break. is_root: bool, ) -> crate::Result<()> { if dependency.behavior.is_optional_peer() { @@ -770,6 +761,46 @@ pub fn enqueue_dependency_with_main_and_success_fn( version_was_replaced = false; break 'version dependency.version.clone(); }; + + // Refuse an unsafe alias (a future folder) or registry name before either is fetched or printed; empty is tolerated. + let alias = this.lockfile.str(&dependency.name); + let alias_is_safe = if alias == this.lockfile.str(&dependency.version.literal) { + // `bun add `'s alias until `assign_resolution` names it: never a folder, but printed. + !dependency::contains_control_character(alias) + } else { + alias.is_empty() || dependency::is_safe_install_folder_name(alias) + }; + let registry_name: &[u8] = match version.tag { + dependency::version::Tag::Npm | dependency::version::Tag::DistTag => { + this.lockfile.str(&name) + } + _ => b"", + }; + let invalid_name = if !alias_is_safe { + Some(alias) + } else if !registry_name.is_empty() && !dependency::is_safe_install_folder_name(registry_name) { + Some(registry_name) + } else { + None + }; + if let Some(invalid_name) = invalid_name { + let name = bun_fmt::escape_control_chars(invalid_name); + if dependency.behavior.is_required() { + this.log_mut().add_error_fmt( + None, + bun_ast::Loc::EMPTY, + format_args!("Invalid dependency name \"{name}\""), + ); + } else { + this.log_mut().add_warning_fmt( + None, + bun_ast::Loc::EMPTY, + format_args!("Invalid dependency name \"{name}\""), + ); + } + return Ok(()); + } + let mut loaded_manifest: Option = None; match version.tag { @@ -797,48 +828,45 @@ pub fn enqueue_dependency_with_main_and_success_fn( Err(err) => { if err == crate::Error::DistTagNotFound { if dependency.behavior.is_required() { - if let Some(fail) = fail_fn { - fail(this, dependency, id, err); - } else if dependency.behavior.is_peer() { - warn_unmet_peer_dependency(this, name, &version); + if dependency.behavior.is_peer() { + warn_unmet_peer_dependency(this, name, &version, err); } else { - this.log_mut() - .add_error_fmt( - None, - bun_ast::Loc::EMPTY, - format_args!( - "Package \"{}\" with tag \"{}\" not found, but package exists", - bstr::BStr::new(this.lockfile.str(&name)), - bstr::BStr::new( - this.lockfile.str(&version.dist_tag().tag) - ), + this.log_mut().add_error_fmt( + None, + bun_ast::Loc::EMPTY, + format_args!( + "Package \"{}\" with tag \"{}\" not found, but package exists", + bun_fmt::escape_control_chars(this.lockfile.str(&name)), + bun_fmt::escape_control_chars( + this.lockfile.str(&version.dist_tag().tag) ), - ); + ), + ); } } return Ok(()); } else if err == crate::Error::NoMatchingVersion { if dependency.behavior.is_required() { - if let Some(fail) = fail_fn { - fail(this, dependency, id, err); - } else if dependency.behavior.is_peer() { - warn_unmet_peer_dependency(this, name, &version); + if dependency.behavior.is_peer() { + warn_unmet_peer_dependency(this, name, &version, err); } else { bun_ast::add_error_pretty!( this.log_mut(), None, bun_ast::Loc::EMPTY, "No version matching \"{}\" found for specifier \"{}\" (but package exists)", - bstr::BStr::new(this.lockfile.str(&version.literal)), - bstr::BStr::new(this.lockfile.str(&name)), + bun_fmt::escape_control_chars( + this.lockfile.str(&version.literal) + ), + bun_fmt::escape_control_chars(this.lockfile.str(&name)), ); } } return Ok(()); } else if err == crate::Error::TooRecentVersion { if dependency.behavior.is_required() { - if let Some(fail) = fail_fn { - fail(this, dependency, id, err); + if dependency.behavior.is_peer() { + warn_unmet_peer_dependency(this, name, &version, err); } else { let age_gate_ms = this.options.minimum_release_age_ms.unwrap_or(0.0); @@ -848,8 +876,10 @@ pub fn enqueue_dependency_with_main_and_success_fn( None, bun_ast::Loc::EMPTY, "Package \"{}\" with tag \"{}\" not found (all versions blocked by minimum-release-age: {} seconds)", - bstr::BStr::new(this.lockfile.str(&name)), - bstr::BStr::new( + bun_fmt::escape_control_chars( + this.lockfile.str(&name) + ), + bun_fmt::escape_control_chars( this.lockfile.str(&version.dist_tag().tag) ), age_gate_ms / MS_PER_S, @@ -860,10 +890,12 @@ pub fn enqueue_dependency_with_main_and_success_fn( None, bun_ast::Loc::EMPTY, "No version matching \"{}\" found for specifier \"{}\" (blocked by minimum-release-age: {} seconds)", - bstr::BStr::new(this.lockfile.str(&name)), - bstr::BStr::new( + bun_fmt::escape_control_chars( this.lockfile.str(&version.literal) ), + bun_fmt::escape_control_chars( + this.lockfile.str(&name) + ), age_gate_ms / MS_PER_S, ); } @@ -872,17 +904,15 @@ pub fn enqueue_dependency_with_main_and_success_fn( return Ok(()); } else if err == crate::Error::MissingPackageJSON { if dependency.behavior.is_required() { - if let Some(fail) = fail_fn { - fail(this, dependency, id, err); - } else if version.tag == dependency::version::Tag::Folder { + if version.tag == dependency::version::Tag::Folder { this.log_mut() .add_error_fmt( None, bun_ast::Loc::EMPTY, format_args!( "Could not find package.json for \"file:{}\" dependency \"{}\"", - bstr::BStr::new(this.lockfile.str(version.folder())), - bstr::BStr::new(this.lockfile.str(&name)), + bun_fmt::escape_control_chars(this.lockfile.str(version.folder())), + bun_fmt::escape_control_chars(this.lockfile.str(&name)), ), ); } else { @@ -891,17 +921,26 @@ pub fn enqueue_dependency_with_main_and_success_fn( bun_ast::Loc::EMPTY, format_args!( "Could not find package.json for dependency \"{}\"", - bstr::BStr::new(this.lockfile.str(&name)), + bun_fmt::escape_control_chars( + this.lockfile.str(&name) + ), ), ); } } return Ok(()); + } else if is_root { + this.log_mut().add_error_fmt( + None, + bun_ast::Loc::EMPTY, + format_args!( + "{} while resolving package \"{}\"", + err.name(), + bun_fmt::escape_control_chars(this.lockfile.str(&name)), + ), + ); + return Ok(()); } else { - if let Some(fail) = fail_fn { - fail(this, dependency, id, err); - return Ok(()); - } return Err(err); } } @@ -916,12 +955,12 @@ pub fn enqueue_dependency_with_main_and_success_fn( bun_core::pretty_errorln!( " -> \"{}\": \"{}\" -> {}@{}", bstr::BStr::new(this.lockfile.str(&result.package.name)), - bstr::BStr::new(label), + bun_fmt::escape_control_chars(label), bstr::BStr::new(this.lockfile.str(&result.package.name)), - result.package.resolution.fmt( + bun_fmt::EscapeControlChars(result.package.resolution.fmt( this.lockfile.buffers.string_bytes.as_slice(), bun_fmt::PathSep::Auto - ), + )), ); } // Resolve dependencies first @@ -1017,12 +1056,17 @@ pub fn enqueue_dependency_with_main_and_success_fn( } if !dependency.behavior.is_peer() || install_peer { - if !this.has_created_network_task( + let needs_extended_manifest = run_tasks::needs_extended_manifest( + this, + dependency.behavior, + task_id, + ); + if !run_tasks::has_created_manifest_task( + this, task_id, dependency.behavior.is_required(), + needs_extended_manifest, ) { - let needs_extended_manifest = - this.options.minimum_release_age_ms.is_some(); if this.options.enable.manifest_cache() { let mut expired = false; // SAFETY: `this_ptr` is the live exclusive @@ -1050,8 +1094,15 @@ pub fn enqueue_dependency_with_main_and_success_fn( // If it's an exact package version already living in the cache // We can skip the network request, even if it's beyond the caching period + // (unless minimum-release-age needs publish times an abbreviated manifest lacks). if version.tag == dependency::version::Tag::Npm && version.npm().version.is_exact() + && (!needs_extended_manifest + || loaded_manifest + .as_ref() + .unwrap() + .pkg + .has_extended_manifest) { if let Some(find_result) = loaded_manifest.as_ref().unwrap().find_by_version( @@ -1068,29 +1119,28 @@ pub fn enqueue_dependency_with_main_and_success_fn( if let Some(min_age_ms) = this.options.minimum_release_age_ms { - if !loaded_manifest - .as_ref() - .unwrap() - .should_exclude_from_age_filter( - this.options.minimum_release_age_excludes, - ) - && Npm::PackageManifest::is_package_version_too_recent( - find_result.package, min_age_ms, - ) + let manifest = + loaded_manifest.as_ref().unwrap(); + let excludes = + this.options.minimum_release_age_excludes; + if !manifest + .should_exclude_from_age_filter(excludes) + && manifest + .is_version_blocked_by_age_filter( + find_result, + min_age_ms, + excludes, + ) { - let package_name = this.lockfile.str(&name); - let min_age_seconds = min_age_ms / MS_PER_S; - let _ = this.log_mut().add_error_fmt( - None, - bun_ast::Loc::EMPTY, - format_args!( - "Version \"{}@{}\" was published within minimum release age of {} seconds", - bstr::BStr::new(package_name), - find_result.version.fmt(this.lockfile.buffers.string_bytes.as_slice()), - min_age_seconds, - ), + // Reported by the `TooRecentVersion` arm above, like a fresh manifest would be. + resolve_result_ = + Err(crate::Error::TooRecentVersion); + run_tasks::manifest_request_not_sent( + this, + task_id, + needs_extended_manifest, ); - return Ok(()); + continue 'retry_with_new_resolve_result; } } // reshaped for borrowck — `find_result` @@ -1121,8 +1171,11 @@ pub fn enqueue_dependency_with_main_and_success_fn( .flatten() { resolve_result_ = Ok(Some(new_resolve_result)); - let _ = - this.network_dedupe_map.remove(&task_id); + run_tasks::manifest_request_not_sent( + this, + task_id, + needs_extended_manifest, + ); continue 'retry_with_new_resolve_result; } } @@ -1131,7 +1184,11 @@ pub fn enqueue_dependency_with_main_and_success_fn( // Was it recent enough to just load it without the network call? if this.options.enable.manifest_cache_control() && !expired { - let _ = this.network_dedupe_map.remove(&task_id); + run_tasks::manifest_request_not_sent( + this, + task_id, + needs_extended_manifest, + ); continue 'retry_from_manifests_ptr; } } @@ -1408,6 +1465,17 @@ pub fn enqueue_dependency_with_main_and_success_fn( dependency::version::Tag::Symlink | dependency::version::Tag::Workspace => { let dependency_tag = version.tag; + if dependency_tag == dependency::version::Tag::Symlink + && !version_was_replaced + && crate::bin::bin_target_escapes_package_dir(this.lockfile.str(version.symlink())) + && let Some((declarer, false)) = this.lockfile.declarer_of(id) + { + if dependency.behavior.is_required() { + reject_escaping_link_of_remote_package(this, declarer, dependency); + } + return Ok(()); + } + let _result = match get_or_put_resolved_package( this, name_hash, @@ -1435,12 +1503,12 @@ pub fn enqueue_dependency_with_main_and_success_fn( bun_core::pretty_errorln!( " -> \"{}\": \"{}\" -> {}@{}", bstr::BStr::new(this.lockfile.str(&result.package.name)), - bstr::BStr::new(label), + bun_fmt::escape_control_chars(label), bstr::BStr::new(this.lockfile.str(&result.package.name)), - result.package.resolution.fmt( + bun_fmt::EscapeControlChars(result.package.resolution.fmt( this.lockfile.buffers.string_bytes.as_slice(), bun_fmt::PathSep::Auto - ), + )), ); } // We shouldn't see any dependencies @@ -1517,6 +1585,16 @@ pub fn enqueue_dependency_with_main_and_success_fn( } dependency::version::Tag::Tarball => { let tarball = version.tarball(); + if matches!(tarball.uri, dependency::tarball::Uri::Local(_)) + && !version_was_replaced + && let Some((declarer, false)) = this.lockfile.declarer_of(id) + && !this.lockfile.has_equal_root_dependency(dependency) + { + if dependency.behavior.is_required() { + reject_local_tarball_of_remote_package(this, declarer, dependency); + } + return Ok(()); + } let res: Resolution = match &tarball.uri { dependency::tarball::Uri::Local(path) => { Resolution::init(ResolutionTagged::LocalTarball(*path)) @@ -1643,20 +1721,89 @@ pub fn enqueue_dependency_with_main_and_success_fn( } /// Unmet peers stay unresolved instead of failing the install; see `may_stay_unresolved`. +/// `err` is the `NoMatchingVersion` / `DistTagNotFound` / `TooRecentVersion` the lookup returned. #[cold] #[inline(never)] fn warn_unmet_peer_dependency( this: &PackageManager, name: SemverString, version: &dependency::Version, + err: crate::Error, ) { - bun_ast::add_warning_pretty!( - this.log_mut(), + let literal = bstr::BStr::new(this.lockfile.str(&version.literal)); + let name = bstr::BStr::new(this.lockfile.str(&name)); + if err == crate::Error::TooRecentVersion { + bun_ast::add_warning_pretty!( + this.log_mut(), + None, + bun_ast::Loc::EMPTY, + "No version matching \"{}\" found for peer dependency \"{}\" (blocked by minimum-release-age: {} seconds)", + literal, + name, + this.options.minimum_release_age_ms.unwrap_or(0.0) / MS_PER_S, + ); + } else { + bun_ast::add_warning_pretty!( + this.log_mut(), + None, + bun_ast::Loc::EMPTY, + "No version matching \"{}\" found for peer dependency \"{}\" (but package exists)", + literal, + name, + ); + } +} + +/// `enqueue_local_tarball` would read the path relative to the project, not to the declarer. +#[cold] +#[inline(never)] +fn reject_local_tarball_of_remote_package( + this: &PackageManager, + declarer: PackageID, + dependency: &Dependency, +) { + let buf = this.lockfile.buffers.string_bytes.as_slice(); + let packages = this.lockfile.packages.slice(); + let name = bstr::BStr::new(dependency.name.slice(buf)); + let literal = bstr::BStr::new(dependency.version.literal.slice(buf)); + let declarer_name = bstr::BStr::new(packages.items_name()[declarer as usize].slice(buf)); + this.log_mut().add_range_error_fmt_with_notes( + None, + bun_ast::Range::NONE, + Box::new([bun_ast::range_data( + None, + bun_ast::Range::NONE, + bun_ast::alloc_print(format_args!( + "add \"{name}\": \"{literal}\" to the root package.json to install that tarball for {declarer_name} as well", + )), + )]), + format_args!( + "refusing to resolve \"{name}@{literal}\" declared by {declarer_name}@{}: local tarball dependencies are only allowed in the package.json files of this project", + packages.items_resolution()[declarer as usize].fmt(buf, bun_fmt::PathSep::Posix), + ), + ); +} + +/// `normalize_package_json_path` resolves the value against the project, not the declaring package, so only package.json files read from the project may use `..` or an absolute path. +#[cold] +#[inline(never)] +fn reject_escaping_link_of_remote_package( + this: &PackageManager, + declarer: PackageID, + dependency: &Dependency, +) { + let buf = this.lockfile.buffers.string_bytes.as_slice(); + let packages = this.lockfile.packages.slice(); + this.log_mut().add_error_fmt( None, bun_ast::Loc::EMPTY, - "No version matching \"{}\" found for peer dependency \"{}\" (but package exists)", - bstr::BStr::new(this.lockfile.str(&version.literal)), - bstr::BStr::new(this.lockfile.str(&name)), + format_args!( + "refusing to resolve \"{}@{}\" declared by {}@{}: link: paths with \"..\" or an absolute path are only allowed in the package.json files of this project", + bstr::BStr::new(dependency.name.slice(buf)), + bstr::BStr::new(dependency.version.literal.slice(buf)), + bstr::BStr::new(packages.items_name()[declarer as usize].slice(buf)), + packages.items_resolution()[declarer as usize].fmt(buf, bun_fmt::PathSep::Posix), + ), ); } @@ -2075,13 +2222,7 @@ fn get_or_put_resolved_package_with_find_result( // borrows `this.lockfile` and `this` at once. Split via raw root. let should_update = this.to_update && if !this.update_requests.is_empty() { - // bun update : every in-scope row (declared or `npm:@…` aliased, see update_scope); other resolutions stay pinned. - let string_buf = this.lockfile.buffers.string_bytes.as_slice(); - (this.is_update_request(dependency.name_hash, dependency.name.slice(string_buf)) - || (name_hash != dependency.name_hash - && this.is_update_request(name_hash, name.slice(string_buf)))) - && crate::update_scope::UpdateScope::of(&*this) - .contains_dependency(&this.lockfile, dependency_id) + is_named_update_row(this, dependency, dependency_id, name_hash, name) } else if let Some(targets) = this.update_target_workspaces.as_deref() { // `bun update -r`/`--filter`: direct deps of the selected workspaces; catalogs are root-scoped. dependency.version.tag == dependency::version::Tag::Catalog @@ -2099,10 +2240,21 @@ fn get_or_put_resolved_package_with_find_result( .is_root_dependency(unsafe { &mut *this_ptr }, dependency_id) }; - // A patched package is held while the range still allows it (update_transitive holds the transitive rows the same way); audit fix does not set to_update and moves it. + // A patched package is held while the range still allows it (audit fix does not set to_update and moves it); + // a row owned by a regular package stays on the copy a direct row (enqueued first) resolves to when its + // range allows, as a fresh resolution would dedupe it there. if should_update && !behavior.is_peer() { - if let Some(id) = patched_package_satisfying(this, name_hash, version) { + let lockfile = &this.lockfile; + let held = if let Some(id) = lockfile.patched_package_satisfying(name_hash, version) { this.kept_patched.push(id); + Some(id) + } else if !lockfile.is_workspace_dependency(dependency_id) { + let direct = |id| lockfile.is_direct_dependency_resolution(id); + lockfile.package_satisfying(name_hash, version, direct) + } else { + None + }; + if let Some(id) = held { success_fn(this, dependency_id, id); return Ok(Some(ResolvedPackageResult { package: *this.lockfile.packages.get(id as usize), @@ -2114,23 +2266,16 @@ fn get_or_put_resolved_package_with_find_result( // Was this package already allocated? Let's reuse the existing one. // - // Determinism: passing `version` here unconditionally lets a - // peer like `>= 1.0.2` collapse onto whichever sibling-appended entry - // (e.g. `1.0.9`) happens to be highest in the index *at this instant* — a - // network-order artefact that the `^1.0.2` peer-hoisting test already - // todoIf's on macOS. The floor guard in `get_package_id` was - // meant to close that, but its exact-pinned/same-major exemptions reopen - // it when *every* candidate is an exact-pinned same-major sibling - // (`uses-a-dep-1..10`). For deferred peers, suppress the satisfies- - // fallback so only an exact `eql(find_result)` can bind here; everything - // else falls through to the `is_peer && !install_peer` defer below and is - // resolved deterministically by phase 2's descending-index scan in - // `get_or_put_resolved_package`. `*` is left alone — it expresses no - // version preference, and the "peer *" hoisting test depends on it - // deduping to whatever sibling pin exists rather than the manifest floor. - let suppress_peer_satisfies = behavior.is_peer() - && !install_peer - && !(version.tag == dependency::version::Tag::Npm && version.npm().version.is_star()); + // A deferred peer may only reuse an exact match: which siblings exist yet depends on arrival + // order, so its range match waits for the `install_peer` pass; likewise only regular rows' pins count. + let suppress_peer_satisfies = behavior.is_peer() && !install_peer; + let pins = !behavior.is_peer() + && version.tag == dependency::version::Tag::Npm + && version.npm().version.is_exact(); + let npm_resolution = Resolution::init(ResolutionTagged::Npm(ResolutionNpmValue { + version: find_result.version, + url: find_result.package.tarball_url.value, + })); if let Some(id) = this.lockfile.get_package_id( name_hash, if should_update || suppress_peer_satisfies { @@ -2138,11 +2283,24 @@ fn get_or_put_resolved_package_with_find_result( } else { Some(version) }, - &Resolution::init(ResolutionTagged::Npm(ResolutionNpmValue { - version: find_result.version, - url: find_result.package.tarball_url.value, - })), + &npm_resolution, ) { + if pins { + this.lockfile.mark_pinned_by_reuse(id); + } + // The existing entry may come from an abbreviated manifest or a pre-`libc` lockfile. + if find_result.package.libc != Npm::Libc::NONE { + let buf = this.lockfile.buffers.string_bytes.as_slice(); + let same_version = this.lockfile.packages.items_resolution()[id as usize].eql( + &npm_resolution, + buf, + buf, + ); + let meta = &mut this.lockfile.packages.items_meta_mut()[id as usize]; + if same_version && meta.libc == Npm::Libc::NONE { + meta.libc = find_result.package.libc; + } + } success_fn(this, dependency_id, id); return Ok(Some(ResolvedPackageResult { package: *this.lockfile.packages.get(id as usize), @@ -2170,15 +2328,9 @@ fn get_or_put_resolved_package_with_find_result( )?)?; debug_assert!(package.meta.id != invalid_package_id); - // Record exact-version pins so `Lockfile::get_package_id`'s - // order-independence guard can tell them apart from range-resolved - // entries (which it treats as network-order artefacts). - if version.tag == dependency::version::Tag::Npm && version.npm().version.is_exact() { - // SAFETY: `this_ptr` is the sole live `&mut PackageManager` here; - // `lockfile.exact_pinned` is disjoint from `package` (returned - // by-value above). - unsafe { &mut *(*this_ptr).lockfile }.mark_exact_pin(package.meta.id); - } + // SAFETY: `this_ptr` is the sole live `&mut PackageManager` here; `package` + // was returned by value above. + unsafe { &mut *(*this_ptr).lockfile }.mark_appended_for(package.meta.id, dependency_id, pins); // Use scopeguard so success_fn runs on every // return below (including the `?` paths). The guard owns the raw pointer so the // `this` reborrow below doesn't conflict with the closure capture. @@ -2198,6 +2350,7 @@ fn get_or_put_resolved_package_with_find_result( let result = match determine_preinstall_state( this, &package, + behavior, &mut name_and_version_hash, &mut patchfile_hash, ) { @@ -2296,107 +2449,16 @@ fn get_or_put_resolved_package( success_fn: SuccessFn, ) -> crate::Result> { if install_peer && behavior.is_peer() { - if let Some(index) = this.lockfile.package_index.get(&name_hash) { - let resolutions = this.lockfile.packages.items_resolution(); - match index { - PackageIndexEntry::Id(existing_id) => { - let existing_id = *existing_id; - if (existing_id as usize) < resolutions.len() { - let existing_resolution = resolutions[existing_id as usize]; - if resolution_satisfies_dependency(this, &existing_resolution, version) { - success_fn(this, dependency_id, existing_id); - return Ok(Some(ResolvedPackageResult { - // we must fetch it from the packages array again, incase the package array mutates the value in the `successFn` - package: *this.lockfile.packages.get(existing_id as usize), - ..Default::default() - })); - } - - let res_tag = resolutions[existing_id as usize].tag; - let ver_tag = version.tag; - if (res_tag == ResolutionTag::Npm - && ver_tag == dependency::version::Tag::Npm) - || (res_tag == ResolutionTag::Git - && ver_tag == dependency::version::Tag::Git) - || (res_tag == ResolutionTag::Github - && ver_tag == dependency::version::Tag::Github) - { - let existing_package = this.lockfile.packages.get(existing_id as usize); - this.log_mut().add_warning_fmt( - None, - bun_ast::Loc::EMPTY, - format_args!( - "incorrect peer dependency \"{}@{}\"", - existing_package - .name - .fmt(this.lockfile.buffers.string_bytes.as_slice()), - existing_package.resolution.fmt( - this.lockfile.buffers.string_bytes.as_slice(), - bun_fmt::PathSep::Auto - ), - ), - ); - success_fn(this, dependency_id, existing_id); - return Ok(Some(ResolvedPackageResult { - // we must fetch it from the packages array again, incase the package array mutates the value in the `successFn` - package: *this.lockfile.packages.get(existing_id as usize), - ..Default::default() - })); - } - } - } - PackageIndexEntry::Ids(list) => { - for &existing_id in list.iter() { - if (existing_id as usize) < resolutions.len() { - let existing_resolution = resolutions[existing_id as usize]; - if resolution_satisfies_dependency(this, &existing_resolution, version) - { - success_fn(this, dependency_id, existing_id); - return Ok(Some(ResolvedPackageResult { - package: *this.lockfile.packages.get(existing_id as usize), - ..Default::default() - })); - } - } - } - - if (list[0] as usize) < resolutions.len() { - let res_tag = resolutions[list[0] as usize].tag; - let ver_tag = version.tag; - if (res_tag == ResolutionTag::Npm - && ver_tag == dependency::version::Tag::Npm) - || (res_tag == ResolutionTag::Git - && ver_tag == dependency::version::Tag::Git) - || (res_tag == ResolutionTag::Github - && ver_tag == dependency::version::Tag::Github) - { - let existing_package_id = list[0]; - let existing_package = - this.lockfile.packages.get(existing_package_id as usize); - this.log_mut().add_warning_fmt( - None, - bun_ast::Loc::EMPTY, - format_args!( - "incorrect peer dependency \"{}@{}\"", - existing_package - .name - .fmt(this.lockfile.buffers.string_bytes.as_slice()), - existing_package.resolution.fmt( - this.lockfile.buffers.string_bytes.as_slice(), - bun_fmt::PathSep::Auto - ), - ), - ); - success_fn(this, dependency_id, list[0]); - return Ok(Some(ResolvedPackageResult { - // we must fetch it from the packages array again, incase the package array mutates the value in the `successFn` - package: *this.lockfile.packages.get(existing_package_id as usize), - ..Default::default() - })); - } - } - } - } + if let Some((existing_id, satisfied)) = + existing_peer_target(this, name_hash, version, dependency_id) + { + return Ok(Some(bind_existing_peer( + this, + dependency_id, + existing_id, + satisfied, + success_fn, + ))); } } @@ -2452,7 +2514,11 @@ fn get_or_put_resolved_package( // materializing `&mut *this_ptr` after `name_str`/`scope` are // derived from it would pop their borrow-stack tags under SB. let cache_ctx = this.manifest_disk_cache_ctx(); - let needs_ext = this.options.minimum_release_age_ms.is_some(); + let needs_ext = run_tasks::needs_extended_manifest( + this, + behavior, + Task::Id::for_manifest(this.lockfile.str(&name)), + ); let this_ptr: *mut PackageManager = this; // SAFETY: `string_bytes` is not resized between here and the // `find_result` lookup; `manifest` lives in `this.manifests` and @@ -2538,7 +2604,7 @@ fn get_or_put_resolved_package( bun_core::pretty_errorln!( "[minimum-release-age] {}@{} selected {} instead of {} due to {}-second filter", bstr::BStr::new(package_name), - bstr::BStr::new(tag_str), + bun_fmt::escape_control_chars(tag_str), result.version.fmt(manifest_buf), newest.fmt(manifest_buf), min_age_seconds, @@ -2546,7 +2612,10 @@ fn get_or_put_resolved_package( } dependency::version::Tag::Npm => { // SAFETY: `version.tag == Npm`. - let version_str = &version.npm().version.fmt(manifest_buf); + let version_str = &version + .npm() + .version + .fmt(this.lockfile.buffers.string_bytes.as_slice()); bun_core::pretty_errorln!( "[minimum-release-age] {}@{} selected {} instead of {} due to {}-second filter", bstr::BStr::new(package_name), @@ -2563,13 +2632,32 @@ fn get_or_put_resolved_package( break 'blk Some(result); } - Npm::FindVersionResult::Err(err_type) => match err_type { - Npm::FindVersionError::TooRecent - | Npm::FindVersionError::AllVersionsTooRecent => { - return Err(crate::Error::TooRecentVersion); + Npm::FindVersionResult::Err(err_type) => { + // The leftover `existing_peer_target` passed over is all there is. + if install_peer && behavior.is_peer() { + if let Some(id) = highest_peer_candidate(&this.lockfile, name_hash, version) + { + return Ok(Some(bind_existing_peer( + this, + dependency_id, + id, + false, + success_fn, + ))); + } } - Npm::FindVersionError::NotFound => None, // Handle below with existing logic - }, + match err_type { + Npm::FindVersionError::TooRecent + | Npm::FindVersionError::AllVersionsTooRecent => { + // The peer pass may still bind it to a same-named package in the tree. + if behavior.is_peer() && !install_peer { + return Ok(None); + } + return Err(crate::Error::TooRecentVersion); + } + Npm::FindVersionError::NotFound => None, // Handle below with existing logic + } + } }; let find_result = match find_result_opt { @@ -2656,7 +2744,24 @@ fn get_or_put_resolved_package( dependency::version::Tag::Folder => { let folder = *version.folder(); let res: FolderResolutionValue = 'res: { - if this.lockfile.is_workspace_dependency(dependency_id) { + if !this.lockfile.is_workspace_dependency(dependency_id) + && crate::bin::bin_target_escapes_package_dir(this.lockfile.str(&folder)) + { + // overrides/resolutions are only ever parsed from the root + // package.json, so a folder path that reached here via an + // override was written by the user and is trusted the same + // as a direct dependency of the root. + let buf = this.lockfile.buffers.string_bytes.as_slice(); + if !this.lockfile.overrides.contains_name( + dependency.name_hash, + dependency.name.slice(buf), + buf, + ) { + break 'res FolderResolutionValue::Err(crate::Error::MissingPackageJSON); + } + } + + if matches!(this.lockfile.declarer_of(dependency_id), Some((_, true))) { // relative to cwd // reshaped for borrowck — `folder_path` borrows // `string_bytes`; detach the slice lifetime so the @@ -2690,22 +2795,8 @@ fn get_or_put_resolved_package( ); } - // transitive folder dependencies do not have their dependencies resolved - if crate::bin::bin_target_escapes_package_dir(this.lockfile.str(&folder)) { - // overrides/resolutions are only ever parsed from the root - // package.json, so a folder path that reached here via an - // override was written by the user and is trusted the same - // as a direct dependency of the root. - let buf = this.lockfile.buffers.string_bytes.as_slice(); - if !this.lockfile.overrides.contains_name( - dependency.name_hash, - dependency.name.slice(buf), - buf, - ) { - break 'res FolderResolutionValue::Err(crate::Error::MissingPackageJSON); - } - } - + // Declared by a registry package: `Package::from_npm` keeps the path + // relative to that package, which is not on disk until it is installed. let mut package = Package::default(); { @@ -2845,8 +2936,24 @@ fn resolved_folder_package( })) } +/// `bun update `: the row names a requested package (declared or `npm:@…` aliased) and is in the update scope; other resolutions stay pinned. +pub(crate) fn is_named_update_row( + this: &PackageManager, + dependency: &Dependency, + dependency_id: DependencyID, + name_hash: PackageNameHash, + name: SemverString, +) -> bool { + let string_buf = this.lockfile.buffers.string_bytes.as_slice(); + (this.is_update_request(dependency.name_hash, dependency.name.slice(string_buf)) + || (name_hash != dependency.name_hash + && this.is_update_request(name_hash, name.slice(string_buf)))) + && crate::update_scope::UpdateScope::of(this) + .contains_dependency(&this.lockfile, dependency_id) +} + /// `--latest` never moves a row below what bun.lock already has (e.g. a prerelease or a version ahead of the tag). -fn keep_locked_if_ahead<'m>( +pub(crate) fn keep_locked_if_ahead<'m>( manifest: &'m Npm::PackageManifest, found: Npm::FindResult<'m>, locked: &Option<(Semver::Version, &[u8])>, @@ -2901,55 +3008,93 @@ fn locked_version_in_lockfile<'a>( name_hash: PackageNameHash, version: &dependency::Version, ) -> Option<(Semver::Version, &'a [u8])> { - if version.tag != dependency::version::Tag::Npm { - return None; - } let lockfile: &Lockfile::Lockfile = &this.lockfile; - let candidates = lockfile.package_index.get(&name_hash)?.as_slice(); - let pkg_res = lockfile.packages.items_resolution(); - let buf = lockfile.buffers.string_bytes.as_slice(); - let range = &version.npm().version; - candidates - .iter() - .copied() - .filter(|&id| id < lockfile.loaded_package_count) - .map(|id| &pkg_res[id as usize]) - .filter(|res| res.tag == ResolutionTag::Npm) - .map(|res| res.npm().version) - .find(|&locked| range.satisfies(locked, buf, buf)) - .map(|locked| (locked, buf)) + let in_lockfile = |id| id < lockfile.loaded_package_count; + let id = lockfile.package_satisfying(name_hash, version, in_lockfile)?; + let res = &lockfile.packages.items_resolution()[id as usize]; + (res.tag == ResolutionTag::Npm) + .then(|| (res.npm().version, lockfile.buffers.string_bytes.as_slice())) } -fn resolution_satisfies_dependency( +/// The package to bind a deferred peer row to and whether it satisfies the row; the highest-or-nothing fallback is what `resolve_peer_dep_version_based` rebinds to on load. +fn existing_peer_target( this: &PackageManager, - resolution: &Resolution, - dependency: &dependency::Version, -) -> bool { - let buf = this.lockfile.buffers.string_bytes.as_slice(); - resolution.satisfies_dependency_version(dependency, buf, buf) + name_hash: PackageNameHash, + version: &dependency::Version, + row: DependencyID, +) -> Option<(PackageID, bool)> { + let lockfile: &Lockfile::Lockfile = &this.lockfile; + if let Some(id) = lockfile.package_satisfying(name_hash, version, |_| true) { + return Some((id, true)); + } + let highest = highest_peer_candidate(lockfile, name_hash, version)?; + (!would_revive_leftover(lockfile, row, highest)).then_some((highest, false)) } -fn patched_package_satisfying( - this: &PackageManager, +/// `package_index` lists the highest version first. +fn highest_peer_candidate( + lockfile: &Lockfile::Lockfile, name_hash: PackageNameHash, version: &dependency::Version, ) -> Option { - let lockfile: &Lockfile::Lockfile = &this.lockfile; - if lockfile.patched_dependencies.count() == 0 { - return None; + let &highest = lockfile.packages_named(name_hash).first()?; + let resolution = lockfile.packages.items_resolution().get(highest as usize)?; + let same_kind = matches!( + (resolution.tag, version.tag), + (ResolutionTag::Npm, dependency::version::Tag::Npm) + | (ResolutionTag::Git, dependency::version::Tag::Git) + | (ResolutionTag::Github, dependency::version::Tag::Github) + ); + same_kind.then_some(highest) +} + +fn bind_existing_peer( + this: &mut PackageManager, + dependency_id: DependencyID, + existing_id: PackageID, + satisfied: bool, + success_fn: SuccessFn, +) -> ResolvedPackageResult { + if !satisfied { + let existing_package = this.lockfile.packages.get(existing_id as usize); + this.log_mut().add_warning_fmt( + None, + bun_ast::Loc::EMPTY, + format_args!( + "incorrect peer dependency \"{}@{}\"", + existing_package + .name + .fmt(this.lockfile.buffers.string_bytes.as_slice()), + bun_core::fmt::for_terminal(existing_package.resolution.fmt( + this.lockfile.buffers.string_bytes.as_slice(), + bun_fmt::PathSep::Auto + )), + ), + ); } - let candidates = lockfile.package_index.get(&name_hash)?.as_slice(); - let pkg_res = lockfile.packages.items_resolution(); - let buf = lockfile.buffers.string_bytes.as_slice(); - candidates.iter().copied().find(|&id| { - let res = &pkg_res[id as usize]; - res.tag == ResolutionTag::Npm - && res.satisfies_dependency_version(version, buf, buf) - && lockfile - .patched_dependencies - .contains(&Semver::string::Builder::string_hash( - &crate::dedupe::label(lockfile, id), - )) + success_fn(this, dependency_id, existing_id); + ResolvedPackageResult { + // we must fetch it from the packages array again, incase the package array mutates the value in the `successFn` + package: *this.lockfile.packages.get(existing_id as usize), + ..Default::default() + } +} + +/// `row` is a root or workspace `peerDependencies` entry and `package_id` is held only by non-root peer rows (usually this entry's own earlier install): nothing provides it, and it is not a root row's copy, which `Tree::hoist_dependency` dedupes every other peer onto regardless of range. +fn would_revive_leftover( + lockfile: &Lockfile::Lockfile, + row: DependencyID, + package_id: PackageID, +) -> bool { + if package_id >= lockfile.loaded_package_count || !lockfile.is_workspace_dependency(row) { + return false; + } + let deps = lockfile.buffers.dependencies.as_slice(); + let resolutions = lockfile.buffers.resolutions.as_slice(); + !(lockfile.packages.items_dependencies().iter().enumerate()).any(|(owner, list)| { + (list.begin() as usize..list.end() as usize).any(|row| { + resolutions[row] == package_id && (owner == 0 || !deps[row].behavior.is_peer()) + }) }) } diff --git a/src/install/PackageManager/PackageManagerLifecycle.rs b/src/install/PackageManager/PackageManagerLifecycle.rs index 63730f9876a4..176b368acddf 100644 --- a/src/install/PackageManager/PackageManagerLifecycle.rs +++ b/src/install/PackageManager/PackageManagerLifecycle.rs @@ -14,6 +14,8 @@ use bun_semver::string::Builder as SemverStringBuilder; use bun_sys as Syscall; use crate::bun_fs::FileSystem; +use crate::bun_json; +use crate::initialize_store; use super::directories; use crate::lifecycle_script_runner::{ @@ -23,7 +25,7 @@ use crate::lockfile_real::package::scripts::List as ScriptsList; use crate::package_manager_real::Command; use crate::resolution_real::Tag as ResolutionTag; use bun_install::lockfile::{Lockfile, Package}; -use bun_install::{PackageID, PackageManager, PreinstallState, invalid_package_id}; +use bun_install::{Behavior, PackageID, PackageManager, PreinstallState, invalid_package_id}; impl PackageManager { pub(crate) fn ensure_preinstall_state_list_capacity(&mut self, count: usize) { @@ -61,9 +63,11 @@ impl PackageManager { /// A separate `lockfile` parameter would always be `manager.lockfile` at every call /// site; collapsed onto `self.lockfile` to avoid the /// `&mut self` / `&self.lockfile` aliasing borrowck rejects. + /// `dependency` is the dependency `pkg` is being resolved for. pub fn determine_preinstall_state( &mut self, pkg: &Package, + dependency: Behavior, out_name_and_version_hash: &mut Option, out_patchfile_hash: &mut Option, ) -> PreinstallState { @@ -71,10 +75,21 @@ impl PackageManager { PreinstallState::Unknown => { // Do not automatically start downloading packages which are disabled // i.e. don't download all of esbuild's versions or SWCs - if pkg.is_disabled(self.options.cpu, self.options.os) { + if !pkg.meta.arch.is_match(self.options.cpu) + || !pkg.meta.os.is_match(self.options.os) + { self.set_preinstall_state(pkg.meta.id, PreinstallState::Done); return PreinstallState::Done; } + // Nor another libc's variant; only for this dependency (`Libc::for_dependency`), + // so the state stays untouched and a later regular dependency still installs it. + if !pkg + .meta + .libc + .is_match(self.options.libc.for_dependency(dependency)) + { + return PreinstallState::Done; + } let patch_hash: Option = 'brk: { if self.lockfile.patched_dependencies.len() == 0 { @@ -140,9 +155,8 @@ impl PackageManager { patch_hash, ) } - ResolutionTag::LocalTarball => directories::cached_tarball_folder_name( - self, - *pkg.resolution.local_tarball(), + ResolutionTag::LocalTarball => directories::cached_local_tarball_folder_name( + &pkg.meta.integrity, patch_hash, ), ResolutionTag::RemoteTarball => directories::cached_tarball_folder_name( @@ -392,6 +406,8 @@ impl PackageManager { path.append(original_path.as_slice())?; script_env.put(b"PATH", path.slice())?; + put_npm_package_config_env(&mut script_env, cwd)?; + // Ownership transfers to `LifecycleScriptSubprocess`, which // re-uses it across every `spawn_next_script` in the chain. Move the // owning `NullDelimitedEnvMap` by value so its `K=V\0` buffers outlive @@ -468,6 +484,38 @@ impl PackageManager { } } +/// The package's own `config` strings as `npm_package_config_`, the subset `bun run` exports. +fn put_npm_package_config_env( + script_env: &mut bun_dotenv::Map, + package_dir: &[u8], +) -> Result<(), crate::Error> { + let package_json_path = join_abs_string_z::(package_dir, &[b"package.json"]); + let Ok(json_buf) = Syscall::File::read_from(Syscall::Fd::cwd(), package_json_path.as_bytes()) + else { + return Ok(()); + }; + let json_src = + bun_ast::Source::init_path_string(package_json_path.as_bytes(), json_buf.as_slice()); + let mut log = bun_ast::Log::init(); + + initialize_store(); + + let Ok(parsed) = bun_json::ParsedJson::parse_package_json(&json_src, &mut log) else { + return Ok(()); + }; + let Some(config) = parsed.root.get(b"config") else { + return Ok(()); + }; + + config.try_for_each_property(|key, _, value| match value.as_utf8_string_literal() { + Some(value) if !key.is_empty() && !value.is_empty() => { + script_env.put(&strings::concat(&[b"npm_package_config_", key]), value) + } + _ => Ok(()), + })?; + Ok(()) +} + fn add_package_to_set( set: &mut ArrayHashMap, lockfile: &Lockfile, @@ -521,8 +569,14 @@ pub fn get_preinstall_state(this: &PackageManager, package_id: PackageID) -> Pre pub fn determine_preinstall_state( this: &mut PackageManager, pkg: &Package, + dependency: Behavior, out_name_and_version_hash: &mut Option, out_patchfile_hash: &mut Option, ) -> PreinstallState { - this.determine_preinstall_state(pkg, out_name_and_version_hash, out_patchfile_hash) + this.determine_preinstall_state( + pkg, + dependency, + out_name_and_version_hash, + out_patchfile_hash, + ) } diff --git a/src/install/PackageManager/PackageManagerOptions.rs b/src/install/PackageManager/PackageManagerOptions.rs index 10504ff5a1cd..a6690ef9611f 100644 --- a/src/install/PackageManager/PackageManagerOptions.rs +++ b/src/install/PackageManager/PackageManagerOptions.rs @@ -1,12 +1,15 @@ use crate::bun_schema::api as Api; +use bun_alloc::{AllocError, Arena}; +use bun_ast::Expr; use bun_core::ZStr; -use bun_core::{Output, env_var}; +use bun_core::{Global, Output, env_var}; use bun_paths::PathBuffer; use super::Subcommand; use super::command_line_arguments::{self, CommandLineArguments}; +use crate::network_task::Authorization; use bun_dotenv::Loader as DotEnvLoader; -use bun_install::{Features, Npm}; +use bun_install::{Behavior, Features, Npm}; // `string` fields are `[]const u8` borrowed from CLI args / bunfig config, // which live for the process lifetime. There is no `deinit` on Options. Mapped to @@ -23,9 +26,13 @@ pub struct Options { pub bin_path: &'static ZStr, pub(crate) did_override_default_scope: bool, + /// `--registry` was given, so `publishConfig.registry` is ignored (as in npm). + pub(crate) registry_from_command_line: bool, pub scope: Npm::registry::Scope, pub(crate) registries: Npm::registry::Map, + /// `.npmrc` `//host/path/` credential lines, resolved by request URL. + pub(crate) url_auth: Vec, pub(crate) cache_directory: &'static [u8], pub enable: Enable, pub do_: Do, @@ -33,6 +40,7 @@ pub struct Options { pub(crate) update: DependencyGroup, pub dry_run: bool, pub check: bool, + pub why: bool, pub(crate) link_workspace_packages: bool, pub(crate) remote_package_features: Features, pub local_package_features: Features, @@ -87,17 +95,32 @@ pub struct Options { // Minimum release age in ms (security feature) // Only install packages published at least N ms ago pub minimum_release_age_ms: Option, - // Packages to exclude from minimum release age checking - pub minimum_release_age_excludes: Option<&'static [&'static [u8]]>, + // Packages and package versions to exclude from minimum release age checking + pub minimum_release_age_excludes: Option<&'static Npm::MinimumReleaseAgeExcludes>, /// Override CPU architecture for optional dependencies filtering pub cpu: Npm::Architecture, /// Override OS for optional dependencies filtering pub os: Npm::OperatingSystem, + /// Override libc for optional dependencies filtering + pub libc: Npm::Libc, pub(crate) config_version: Option, } +impl Options { + /// Only the full registry document has the publish times `minimumReleaseAge` filters on. + pub(crate) fn needs_extended_manifest_to_pick_versions(&self) -> bool { + self.minimum_release_age_ms.is_some() + } + + /// The abbreviated document also lacks `libc`, which is only enforced for optional + /// dependencies (`Libc::for_dependency`); host-independent so lockfiles stay portable. + pub(crate) fn needs_extended_manifest(&self, dependency: Behavior) -> bool { + self.needs_extended_manifest_to_pick_versions() || dependency.is_optional() + } +} + impl Default for Options { fn default() -> Self { Self { @@ -107,9 +130,11 @@ impl Default for Options { explicit_global_directory: b"", bin_path: bun_paths::path_literal!("node_modules/.bin"), did_override_default_scope: false, + registry_from_command_line: false, // Always assigned in `load()` before read. scope: Npm::registry::Scope::default(), registries: Npm::registry::Map::default(), + url_auth: Vec::new(), cache_directory: b"", enable: Enable::default(), do_: Do::default(), @@ -117,6 +142,7 @@ impl Default for Options { update: DependencyGroup::default(), dry_run: false, check: false, + why: false, link_workspace_packages: true, remote_package_features: Features { optional_dependencies: true, @@ -161,6 +187,7 @@ impl Default for Options { minimum_release_age_excludes: None, cpu: Npm::Architecture::CURRENT, os: Npm::OperatingSystem::CURRENT, + libc: Npm::Libc::CURRENT, config_version: None, } } @@ -258,6 +285,206 @@ impl Options { _ => &self.scope, } } + + /// `scope`'s credentials only go to its own origin (the registry controls `dist.tarball`); + /// any other origin gets exactly what `.npmrc` configures for it, or nothing. + pub(crate) fn tarball_credentials<'a>( + &'a self, + scope: &'a Npm::registry::Scope, + tarball: &bun_url::URL, + ) -> Option<&'a Npm::registry::Scope> { + if scope.has_credentials() && is_same_origin(tarball, &scope.url.url()) { + return Some(scope); + } + Npm::registry::UrlAuth::find(&self.url_auth, tarball) + } + + /// Appended to the `GET - 401` line of a request bun sent without `Authorization`: + /// which `.npmrc` line would have supplied one. Empty when credentials were sent. + pub(crate) fn missing_credentials_note( + &self, + package_name: &[u8], + url: &[u8], + request: RequestKind, + ) -> Vec { + use std::io::Write as _; + + let scope = self.scope_for_package_name(package_name); + let mut note = Vec::new(); + match request { + RequestKind::Manifest if !scope.has_credentials() => { + let registry = scope.url.url(); + let path = registry + .pathname + .strip_suffix(b"/") + .unwrap_or(registry.pathname); + let _ = write!( + note, + "\n no credentials are configured for this registry; add //{}{}/:_authToken= to .npmrc", + bstr::BStr::new(registry.host), + bstr::BStr::new(path), + ); + } + RequestKind::Tarball(Authorization::AllowAuthorization) => { + let url = bun_url::URL::parse(url); + if self.tarball_credentials(scope, &url).is_some() { + return note; + } + if scope.has_credentials() { + let _ = write!( + note, + "\n the credentials configured for {} are not sent to {}; add //{}/:_authToken= to .npmrc if this host needs them", + bstr::BStr::new(scope.url.url().host), + bstr::BStr::new(url.host), + bstr::BStr::new(url.host), + ); + } else { + let _ = write!( + note, + "\n no credentials are configured for {}; add //{}/:_authToken= to .npmrc", + bstr::BStr::new(url.host), + bstr::BStr::new(url.host), + ); + } + } + _ => {} + } + note + } + + /// How `--registry` and `$NPM_CONFIG_REGISTRY` pick up a `//host/:_authToken=` line. + fn fill_credentials_from_url_auth(&mut self) { + let url_auth = &self.url_auth; + for scope in core::iter::once(&mut self.scope).chain(self.registries.values_mut()) { + if !scope.has_credentials() { + if let Some(found) = Npm::registry::UrlAuth::find(url_auth, &scope.url.url()) { + scope.copy_credentials_from(found); + } + } + } + } +} + +/// Component-wise, so `https://host:443/` in `dist.tarball` matches `https://host/` in `.npmrc`. +fn is_same_origin(a: &bun_url::URL, b: &bun_url::URL) -> bool { + a.protocol.eq_ignore_ascii_case(b.protocol) + && a.hostname.eq_ignore_ascii_case(b.hostname) + && a.get_port_auto() == b.get_port_auto() +} + +#[derive(Clone, Copy)] +pub(crate) enum RequestKind { + Manifest, + /// What the request was enqueued with (`NetworkTask::authorization`). + Tarball(Authorization), +} + +impl Options { + /// The scope for `url` once it replaces `current`, with the credentials configured for `url`. + fn scope_for_registry_url( + &self, + name: &[u8], + current: &Npm::registry::Scope, + url: &[u8], + ) -> Npm::registry::Scope { + let mut scope = Npm::registry::Scope { + name: name.into(), + ..Default::default() + }; + scope.set_url(url.into()); + let configured = core::iter::once(&self.scope) + .chain(self.registries.values()) + .find(|configured| configured.url_hash == scope.url_hash) + .or_else(|| Npm::registry::UrlAuth::find(&self.url_auth, &scope.url.url())); + if let Some(configured) = configured { + scope.copy_credentials_from(configured); + return scope; + } + // Unconfigured `url`: `current`'s credentials follow it only same-host, and never to http. + let (new_url, current_url) = (scope.url.url(), current.url.url()); + if bun_core::without_trailing_slash(new_url.host) + == bun_core::without_trailing_slash(current_url.host) + && (new_url.is_https() || !current_url.is_https()) + { + scope.copy_credentials_from(current); + } + scope + } + + fn set_default_registry(&mut self, url: &[u8]) { + self.scope = self.scope_for_registry_url(b"", &self.scope, url); + self.did_override_default_scope = self.scope.url_hash != *Npm::registry::DEFAULT_URL_HASH; + } + + /// Applies the `publishConfig` of the package being published; command-line flags win over it. + pub fn apply_publish_config( + &mut self, + package_json: &Expr, + bump: &Arena, + package_name: &[u8], + ) -> Result<(), AllocError> { + let Some(config) = package_json.get(b"publishConfig") else { + return Ok(()); + }; + + if self.publish_config.tag.is_empty() { + if let Some(tag) = config.get_string_cloned(bump, b"tag")? { + self.publish_config.tag = leak_static(tag); + } + } + + if self.publish_config.access.is_none() { + if let Some(access) = config.get_string_cloned(bump, b"access")? { + self.publish_config.access = Some(Access::from_str(access).unwrap_or_else(|| { + Output::err_generic("invalid `access` value: '{}'", (bstr::BStr::new(access),)); + Global::crash(); + })); + } + } + + // As in npm, `registry` replaces only the default registry; `@scope:registry` the scope's. + if !self.registry_from_command_line { + if let Some(url) = publish_config_registry(&config, bump, b"registry")? { + self.set_default_registry(url); + } + } + + if package_name.starts_with(b"@") { + let scope_name = Npm::registry::Scope::get_name(package_name); + let key = [b"@".as_slice(), scope_name, b":registry"].concat(); + if let Some(url) = publish_config_registry(&config, bump, &key)? { + let current = self.scope_for_package_name(package_name); + let scope = self.scope_for_registry_url(scope_name, current, url); + self.registries + .put(Npm::registry::Scope::hash(scope_name), scope)?; + } + } + + Ok(()) + } +} + +/// `publishConfig[key]` as an http(s) URL; anything else errors rather than publishing elsewhere. +fn publish_config_registry<'b>( + config: &Expr, + bump: &'b Arena, + key: &[u8], +) -> Result, AllocError> { + let Some(value) = config.get(key) else { + return Ok(None); + }; + match value.as_string_cloned(bump)? { + Some(url) if url.starts_with(b"https://") || url.starts_with(b"http://") => Ok(Some(url)), + not_a_url => { + let got = + not_a_url.map_or_else(String::new, |v| format!(": {}", bun_core::fmt::quote(v))); + Output::err_generic( + "invalid `{}` value in `publishConfig`{}, expected a URL starting with 'https://' or 'http://'", + (bstr::BStr::new(key), got), + ); + Global::crash(); + } + } } #[derive(Copy, Clone, PartialEq, Eq, Default, Debug)] @@ -298,9 +525,23 @@ pub use crate::config_version::ConfigVersion; pub use bun_install_types::DependencyGroup; pub use bun_install_types::NodeLinker::NodeLinker; +/// mkdir -p + open `/`; `base` is an environment value of any length. +fn make_open_dir_under(base: &[u8], parts: &[&[u8]]) -> crate::Result { + use bun_paths::{platform, resolve_path::join_abs_string_buf_checked}; + use bun_sys::{Dir, OpenDirOptions}; + + let mut buf = PathBuffer::uninit(); + let Some(path) = join_abs_string_buf_checked::(base, &mut buf.0, parts) else { + return Err(crate::Error::Sys(bun_errno::SystemErrno::ENAMETOOLONG)); + }; + Dir::cwd() + .make_open_path(path, OpenDirOptions::default()) + .map(|d| d.into_raw()) + .map_err(Into::into) +} + // mkdir -p + open the dir. Callers store the raw `Fd` (`options.global_bin_dir: Fd`). pub fn open_global_dir(explicit_global_dir: &[u8]) -> crate::Result { - use bun_paths::{platform, resolve_path::join_abs_string_buf}; use bun_sys::{Dir, OpenDirOptions}; if let Some(home_dir) = env_var::BUN_INSTALL_GLOBAL_DIR.get() { @@ -318,33 +559,20 @@ pub fn open_global_dir(explicit_global_dir: &[u8]) -> crate::Result } if let Some(home_dir) = env_var::BUN_INSTALL.get() { - let mut buf = PathBuffer::uninit(); - let parts: [&[u8]; 2] = [b"install", b"global"]; - let path = join_abs_string_buf::(home_dir, &mut buf.0, &parts); - return Dir::cwd() - .make_open_path(path, OpenDirOptions::default()) - .map(|d| d.into_raw()) - .map_err(Into::into); + return make_open_dir_under(home_dir, &[b"install", b"global"]); } if let Some(home_dir) = env_var::XDG_CACHE_HOME .get() .or_else(|| env_var::HOME.get()) { - let mut buf = PathBuffer::uninit(); - let parts: [&[u8]; 3] = [b".bun", b"install", b"global"]; - let path = join_abs_string_buf::(home_dir, &mut buf.0, &parts); - return Dir::cwd() - .make_open_path(path, OpenDirOptions::default()) - .map(|d| d.into_raw()) - .map_err(Into::into); + return make_open_dir_under(home_dir, &[b".bun", b"install", b"global"]); } Err(crate::Error::NoGlobalDirectoryFound) } pub(crate) fn open_global_bin_dir(opts_: Option<&Api::BunInstall>) -> crate::Result { - use bun_paths::{platform, resolve_path::join_abs_string_buf}; use bun_sys::{Dir, OpenDirOptions}; if let Some(home_dir) = env_var::BUN_INSTALL_BIN.get() { @@ -366,26 +594,14 @@ pub(crate) fn open_global_bin_dir(opts_: Option<&Api::BunInstall>) -> crate::Res } if let Some(home_dir) = env_var::BUN_INSTALL.get() { - let mut buf = PathBuffer::uninit(); - let parts: [&[u8]; 1] = [b"bin"]; - let path = join_abs_string_buf::(home_dir, &mut buf.0, &parts); - return Dir::cwd() - .make_open_path(path, OpenDirOptions::default()) - .map(|d| d.into_raw()) - .map_err(Into::into); + return make_open_dir_under(home_dir, &[b"bin"]); } if let Some(home_dir) = env_var::XDG_CACHE_HOME .get() .or_else(|| env_var::HOME.get()) { - let mut buf = PathBuffer::uninit(); - let parts: [&[u8]; 2] = [b".bun", b"bin"]; - let path = join_abs_string_buf::(home_dir, &mut buf.0, &parts); - return Dir::cwd() - .make_open_path(path, OpenDirOptions::default()) - .map(|d| d.into_raw()) - .map_err(Into::into); + return make_open_dir_under(home_dir, &[b".bun", b"bin"]); } Err(crate::Error::MissingGlobalBinDirectoryTrySettingBUNINSTALL) @@ -450,6 +666,12 @@ impl Options { } } + for url_auth in &config.url_auth { + if let Some(url_auth) = Npm::registry::UrlAuth::from_api(url_auth, env)? { + self.url_auth.push(url_auth); + } + } + if let Some(ca) = &config.ca { match ca { Api::Ca::List(ca_list) => { @@ -564,8 +786,9 @@ impl Options { exclusions.iter().map(|e| leak_static(e)).collect(); // Parked for the lifetime of the install command (config arena // equivalent), same as `leak_static` above. - self.minimum_release_age_excludes = - Some(&*bun_core::heap::release(leaked.into_boxed_slice())); + self.minimum_release_age_excludes = Some(&*bun_core::heap::release(Box::new( + Npm::MinimumReleaseAgeExcludes::parse(&leaked, log), + ))); } // `PnpmMatcher` is move-only; `config` is `&` here so the matchers @@ -632,20 +855,9 @@ impl Options { if api_registry.has_credentials() { self.scope = Npm::registry::Scope::from_api(b"", api_registry, env)?; } else { - let new_url = bun_url::URL::parse(&api_registry.url); - let same_origin = { - let prev_url = self.scope.url.url(); - bun_core::without_trailing_slash(new_url.host) - == bun_core::without_trailing_slash(prev_url.host) - && (new_url.is_https() || !prev_url.is_https()) - }; - if !same_origin { - self.scope.token = Box::default(); - self.scope.auth = Box::default(); - self.scope.user = Box::default(); - } - self.scope.set_url(api_registry.url); + self.set_default_registry(&api_registry.url); } + self.registry_from_command_line = true; } } @@ -691,7 +903,7 @@ impl Options { } if let Some(check_bool) = env.get(b"BUN_CONFIG_NO_VERIFY") { - self.do_.set(Do::VERIFY_INTEGRITY, check_bool != b"0"); + self.do_.set(Do::VERIFY_INTEGRITY, check_bool == b"0"); } // Update should never read from manifest cache @@ -729,6 +941,7 @@ impl Options { self.do_.set(Do::SAVE_LOCKFILE, false); } self.check = cli.check; + self.why = cli.why; if cli.no_summary || cli.log_level.is_silent() { self.do_.set(Do::SUMMARY, false); @@ -817,9 +1030,10 @@ impl Options { .store(backend as u8, core::sync::atomic::Ordering::Relaxed); } - // CPU and OS are now parsed as enums in CommandLineArguments, just copy them + // CPU, OS and libc are now parsed as enums in CommandLineArguments, just copy them self.cpu = cli.cpu; self.os = cli.os; + self.libc = cli.libc; self.do_.set(Do::UPDATE_TO_LATEST, cli.latest); self.do_.set(Do::RECURSIVE, cli.recursive); @@ -905,6 +1119,8 @@ impl Options { self.enable.set(Enable::FORCE_SAVE_LOCKFILE, false); } + self.fill_credentials_from_url_auth(); + // moved from `defer { ... }` after scope assignment (see note above). self.did_override_default_scope = self.scope.url_hash != *Npm::registry::DEFAULT_URL_HASH; diff --git a/src/install/PackageManager/PackageManagerResolution.rs b/src/install/PackageManager/PackageManagerResolution.rs index 962859afb17f..740273a6207e 100644 --- a/src/install/PackageManager/PackageManagerResolution.rs +++ b/src/install/PackageManager/PackageManagerResolution.rs @@ -140,23 +140,8 @@ impl PackageManager { if !parsed.valid || parsed.wildcard != semver::query::Wildcard::None { continue; } - // not handling OOM // TODO: wildcard - let mut version = parsed.version.min(); - let total = (version.tag.build.len() + version.tag.pre.len()) as usize; - if total > 0 { - let len_before = tags_buf.len(); - // `clone_into` writes exactly `total` bytes (build.len + pre.len) - // into `available` and advances it; zero-fill the tail first so - // we can hand it out as a safe `&mut [u8]` instead of slicing - // raw spare capacity. - tags_buf.resize(len_before + total, 0); - let mut available = &mut tags_buf[len_before..]; - let new_version = version.clone_into(name, &mut available); - version = new_version; - } - - list.push(version); + list.push(parsed.version.min().clone_into(name, tags_buf)); } Ok(list) @@ -333,11 +318,10 @@ impl PackageManager { continue; } - let features = match pkg_resolutions[parent_id].tag { - ResolutionTag::Root | ResolutionTag::Workspace | ResolutionTag::Folder => { - self.options.local_package_features - } - _ => self.options.remote_package_features, + let features = if pkg_resolutions[parent_id].tag.is_local_package() { + self.options.local_package_features + } else { + self.options.remote_package_features }; // even if optional dependencies are enabled, it's still allowed to fail if failed_dep.behavior.is_optional() || !failed_dep.behavior.is_enabled(features) { @@ -391,14 +375,20 @@ impl PackageManager { { Output::err_generic( "{} failed to resolve", - (failed_dep.version.literal.fmt(string_buf),), + (bun_core::fmt::for_terminal( + failed_dep.version.literal.fmt(string_buf), + ),), ); } else { Output::err_generic( "{}@{} failed to resolve", ( - bstr::BStr::new(failed_dep.name.slice(string_buf)), - failed_dep.version.literal.fmt(string_buf), + bun_core::fmt::escape_control_chars( + failed_dep.name.slice(string_buf), + ), + bun_core::fmt::for_terminal( + failed_dep.version.literal.fmt(string_buf), + ), ), ); } diff --git a/src/install/PackageManager/PopulateManifestCache.rs b/src/install/PackageManager/PopulateManifestCache.rs index 92750c35cc3c..0ceb71398ccc 100644 --- a/src/install/PackageManager/PopulateManifestCache.rs +++ b/src/install/PackageManager/PopulateManifestCache.rs @@ -50,7 +50,8 @@ fn start_manifest_task( needs_extended_manifest: bool, ) -> Result<(), StartManifestTaskError> { let task_id = Task::Id::for_manifest(pkg_name); - if run_tasks::has_created_network_task(manager, task_id, is_required) { + if run_tasks::has_created_manifest_task(manager, task_id, is_required, needs_extended_manifest) + { return Ok(()); } if manager.options.log_level.show_progress() { @@ -94,6 +95,8 @@ fn start_manifest_task( #[derive(Clone, Copy)] pub enum Packages<'a> { /// Every npm package in the lockfile; best-effort (the post-migration backfill), so failures are warnings. + /// The only mode that records platform fields, so the only one that requests the extended document + /// for optional dependencies (`Options::needs_extended_manifest`); the other two just pick versions. All, /// The direct dependencies of these workspace packages; a required one failing is an error, see [`print_fetch_failures`]. Ids(&'a [PackageID]), @@ -149,9 +152,14 @@ pub fn populate_manifest_cache( match packages { Packages::All => { - let mut seen_pkg_ids: HashMap = HashMap::new(); + // Value: whether the extended document was already requested for the + // package. A package that is a regular dependency of one package and an + // optional dependency of another needs the extended one whichever of its + // dependencies comes first (`has_created_manifest_task` dedupes the + // requests themselves). + let mut seen_pkg_ids: HashMap = HashMap::new(); - for _dep_id in 0..dependencies.len() { + for (_dep_id, dep) in dependencies.iter().enumerate() { let dep_id: DependencyID = DependencyID::try_from(_dep_id).expect("int cast"); let pkg_id = resolutions[dep_id as usize]; @@ -159,10 +167,15 @@ pub fn populate_manifest_cache( continue; } - // `getOrPut(pkg_id).found_existing` — value is `void`, so this is a set insert. - if seen_pkg_ids.insert(pkg_id, ()).is_some() { + // `options` is not mutated between here and the + // `start_manifest_task` call — read via the BACKREF `mgr_ref`. + let needs_extended_manifest = mgr_ref.options.needs_extended_manifest(dep.behavior); + + let seen = seen_pkg_ids.get_or_put(pkg_id)?; + if seen.found_existing && (*seen.value_ptr || !needs_extended_manifest) { continue; } + *seen.value_ptr = needs_extended_manifest; let res = &pkg_resolutions[pkg_id as usize]; if res.tag != ResolutionTag::Npm { @@ -171,9 +184,6 @@ pub fn populate_manifest_cache( let pkg_name = pkg_names[pkg_id as usize]; let pkg_name_slice = pkg_name.slice(string_buf); - // `options` is not mutated between here and the - // `start_manifest_task` call — read via the BACKREF `mgr_ref`. - let needs_extended_manifest = mgr_ref.options.minimum_release_age_ms.is_some(); // `scope_for_package_name` borrows only `options` (via the // BACKREF `mgr_ref`); `manifests` is a disjoint field projected @@ -232,7 +242,8 @@ pub fn populate_manifest_cache( // `options` read via BACKREF `mgr_ref` — see provenance-root // note above. - let needs_extended_manifest = mgr_ref.options.minimum_release_age_ms.is_some(); + let needs_extended_manifest = + mgr_ref.options.needs_extended_manifest_to_pick_versions(); let package_name = pkg_names[pkg_id as usize].slice(string_buf); // See disjoint-field note on the `.All` arm above. let scope = @@ -268,12 +279,13 @@ pub fn populate_manifest_cache( } } Packages::Exact(ids) => { + let needs_extended_manifest = + mgr_ref.options.needs_extended_manifest_to_pick_versions(); for &pkg_id in ids { if pkg_resolutions[pkg_id as usize].tag != ResolutionTag::Npm { continue; } let package_name = pkg_names[pkg_id as usize].slice(string_buf); - let needs_extended_manifest = mgr_ref.options.minimum_release_age_ms.is_some(); let scope = bun_ptr::BackRef::new(mgr_ref.options.scope_for_package_name(package_name)); // SAFETY: `manifests` is disjoint from `options`/`lockfile`; `manager_ptr` is the SRW root. diff --git a/src/install/PackageManager/ProgressStrings.rs b/src/install/PackageManager/ProgressStrings.rs index 5ba05d1890fa..d1f5825991ac 100644 --- a/src/install/PackageManager/ProgressStrings.rs +++ b/src/install/PackageManager/ProgressStrings.rs @@ -1,6 +1,6 @@ use core::sync::atomic::Ordering; -use bun_core::Output; +use bun_core::{Output, strings}; use const_format::concatcp; use crate::bun_progress::Node as ProgressNode; @@ -94,24 +94,25 @@ impl PackageManager { name: &[u8], emoji: &[u8], ) { + let emoji_len = if Output::enable_ansi_colors_stderr() { + if IS_FIRST { + self.progress_name_buf[..emoji.len()].copy_from_slice(emoji); + } + emoji.len() + } else { + 0 + }; + // Display-only, so a name longer than the buffer is simply cut short. + let name = + strings::truncate_to_char_boundary(name, self.progress_name_buf.len() - emoji_len); + self.progress_name_buf[emoji_len..][..name.len()].copy_from_slice(name); + let len = emoji_len + name.len(); // SAFETY: `node` is `self.downloads_node` / `self.scripts_node`, both of // which point at storage owned by (or outliving) this `PackageManager` // singleton; `progress_name_buf` is an inline field of that same // singleton, so the buffer outlives every node that references it and // erasing the slice lifetime to `'static` is sound. - unsafe { - let len = if Output::enable_ansi_colors_stderr() { - if IS_FIRST { - self.progress_name_buf[..emoji.len()].copy_from_slice(emoji); - } - self.progress_name_buf[emoji.len()..][..name.len()].copy_from_slice(name); - emoji.len() + name.len() - } else { - self.progress_name_buf[..name.len()].copy_from_slice(name); - name.len() - }; - node.name = bun_ptr::detach_lifetime(&self.progress_name_buf[..len]); - } + node.name = unsafe { bun_ptr::detach_lifetime(&self.progress_name_buf[..len]) }; } pub fn start_progress_bar_if_none(&mut self) { diff --git a/src/install/PackageManager/WorkspacePackageJSONCache.rs b/src/install/PackageManager/WorkspacePackageJSONCache.rs index 3ef125dddb95..cef56492e5fc 100644 --- a/src/install/PackageManager/WorkspacePackageJSONCache.rs +++ b/src/install/PackageManager/WorkspacePackageJSONCache.rs @@ -114,14 +114,36 @@ pub enum GetResult<'a> { ParseErr(Error), } +/// The step of [`WorkspacePackageJSONCache::get_with_path`] that failed. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum GetStep { + Read, + Parse, +} + +impl GetStep { + /// The word for the step in a "failed to ... package.json" message. + pub fn verb(self) -> &'static str { + match self { + GetStep::Read => "read", + GetStep::Parse => "parse", + } + } +} + impl<'a> GetResult<'a> { - pub(crate) fn unwrap(self) -> Result<&'a mut MapEntry, Error> { + /// The entry, or the step that failed and its error. + pub fn entry(self) -> Result<&'a mut MapEntry, (GetStep, Error)> { match self { GetResult::Entry(entry) => Ok(entry), - GetResult::ReadErr(err) => Err(err), - GetResult::ParseErr(err) => Err(err), + GetResult::ReadErr(err) => Err((GetStep::Read, err)), + GetResult::ParseErr(err) => Err((GetStep::Parse, err)), } } + + pub(crate) fn unwrap(self) -> Result<&'a mut MapEntry, Error> { + self.entry().map_err(|(_, err)| err) + } } #[derive(Default)] diff --git a/src/install/PackageManager/add_remove_with_filter.rs b/src/install/PackageManager/add_remove_with_filter.rs index 1ae652f46c3e..bcda0f4430a6 100644 --- a/src/install/PackageManager/add_remove_with_filter.rs +++ b/src/install/PackageManager/add_remove_with_filter.rs @@ -298,10 +298,14 @@ pub(crate) fn write_target(manager: &mut PackageManager, target: &WorkspaceTarge match File::write_file(Fd::cwd(), path, &entry.source.contents) { Ok(()) => true, Err(err) => { - Output::err_generic( - "failed to write package.json for workspace '{}': {}", - (BStr::new(&target.name), BStr::new(err.name())), - ); + if target.name.is_empty() { + Output::err_generic("failed to write package.json: {}", (BStr::new(err.name()),)); + } else { + Output::err_generic( + "failed to write package.json for workspace '{}': {}", + (BStr::new(&target.name), BStr::new(err.name())), + ); + } false } } diff --git a/src/install/PackageManager/install_with_manager.rs b/src/install/PackageManager/install_with_manager.rs index 8fb503659cec..db9210636e8f 100644 --- a/src/install/PackageManager/install_with_manager.rs +++ b/src/install/PackageManager/install_with_manager.rs @@ -1,7 +1,7 @@ use core::sync::atomic::Ordering; use bun_ast::Source; -use bun_collections::{DynamicBitSet, index_sort}; +use bun_collections::DynamicBitSet; use bun_core::UnwrapOrOom as _; use bun_core::time::nano_timestamp; use bun_core::{Global, Output}; @@ -24,7 +24,7 @@ use crate::{ }; // Bring the typed `items_()` column accessors into scope for // `MultiArrayList` / `Slice`. -use super::Command; +use super::{Command, PackageJSONEditor}; use crate::PackageManager; use crate::config_version::ConfigVersion; use crate::hoisted_install::install_hoisted_packages; @@ -39,6 +39,8 @@ use bun_install_types::NodeLinker::NodeLinker; // Free-function "methods" on `PackageManager` hosted in sibling modules // to avoid one giant `impl PackageManager` block. +use crate::package_manager_real::enqueue::add_dependency_error; +use crate::package_manager_real::remove_stale_workspace_links::remove_stale_workspace_links; use crate::package_manager_real::run_tasks::{RunTasksCallbacks, run_tasks}; use crate::package_manager_real::{ UpdateRequest, enqueue_dependency_list, enqueue_dependency_with_main, enqueue_patch_task_pre, @@ -92,7 +94,7 @@ pub fn install_with_manager( // Snapshot the loaded-from-lockfile package count so // `Lockfile::get_package_id` can tell loaded pins apart from packages // appended by manifest fetches in this resolve session. - manager.lockfile.mark_loaded_packages(); + manager.lockfile.mark_loaded_packages()?; let (config_version, changed_config_version) = load_result.choose_config_version(); manager.options.config_version = Some(config_version); @@ -236,7 +238,7 @@ pub fn install_with_manager( if manager.subcommand == Subcommand::Dedupe { crate::dedupe::dedupe_after_differ(manager); } - if manager.summary.changes_resolutions() { + if manager.summary.changes_resolutions() || bare_update { direct_deps_before = DirectDependencies::snapshot(&manager.lockfile); } @@ -326,7 +328,7 @@ pub fn install_with_manager( let mut v = Vec::new(); lf.overrides.append_overridden_name_hashes(&mut v); lockfile.overrides.append_overridden_name_hashes(&mut v); - index_sort::sort_slice_unstable_by(&mut v, |a, b| a.cmp(b)); + v.sort_unstable(); v.dedup(); v }; @@ -484,72 +486,26 @@ pub fn install_with_manager( pinned_rows = enqueue_transitive(manager, &transitive, invalidates_rows)?; } - // `enqueueDependencyWithMain` can reach `Lockfile.Package.fromNPM`, - // which grows `buffers.dependencies` and may reallocate it. - // Iterate by index against a snapshot of the original length and - // copy each entry to the stack so neither the loop nor the callee - // ever reads through a pointer into the old backing storage. - if manager.summary.overrides_changed && !all_name_hashes.is_empty() { - let dependencies_len = manager.lockfile.buffers.dependencies.len(); - for dependency_i in 0..dependencies_len { - if pinned_rows.is_set_allow_out_of_bound(dependency_i, false) { - continue; - } - let dependency = - manager.lockfile.buffers.dependencies[dependency_i].clone(); - if all_name_hashes.binary_search(&dependency.name_hash).is_ok() { - manager.lockfile.buffers.resolutions[dependency_i] = - invalid_package_id; - if let Err(err) = enqueue_dependency_with_main( - manager, - dependency_i as u32, - &dependency, - invalid_package_id, - false, - ) { - add_dependency_error(manager, &dependency, err); - } - } - } - } - - if manager.summary.catalogs_changed { + if invalidates_rows { + let catalogs_changed = manager.summary.catalogs_changed; let mut catalog_overridden: Vec = Vec::new(); - manager - .lockfile - .overrides - .append_catalog_valued_name_hashes(&mut catalog_overridden); - index_sort::sort_slice_unstable_by(&mut catalog_overridden, |a, b| { - a.cmp(b) - }); - catalog_overridden.dedup(); - let dependencies_len = manager.lockfile.buffers.dependencies.len(); - for _dep_id in 0..dependencies_len { - let dep_id: DependencyID = u32::try_from(_dep_id).expect("int cast"); - if pinned_rows.is_set_allow_out_of_bound(_dep_id, false) { - continue; - } - let dep = - manager.lockfile.buffers.dependencies[dep_id as usize].clone(); - if dep.version.tag != DependencyVersionTag::Catalog - && (catalog_overridden.is_empty() - || catalog_overridden.binary_search(&dep.name_hash).is_err()) - { - continue; - } - - manager.lockfile.buffers.resolutions[dep_id as usize] = - invalid_package_id; - if let Err(err) = enqueue_dependency_with_main( - manager, - dep_id, - &dep, - invalid_package_id, - false, - ) { - add_dependency_error(manager, &dep, err); - } + if catalogs_changed { + manager + .lockfile + .overrides + .append_catalog_valued_name_hashes(&mut catalog_overridden); + catalog_overridden.sort_unstable(); + catalog_overridden.dedup(); } + // `all_name_hashes` is empty unless the overrides changed. + reresolve_owned_rows(manager, &pinned_rows, |dependency| { + all_name_hashes.binary_search(&dependency.name_hash).is_ok() + || (catalogs_changed + && dependency.version.tag == DependencyVersionTag::Catalog) + || catalog_overridden + .binary_search(&dependency.name_hash) + .is_ok() + }); } // Split this into two passes because the below may allocate memory or invalidate pointers @@ -595,6 +551,9 @@ pub fn install_with_manager( let mut named = NamedUpdates::default(); if !needs_new_lockfile { if named_update { + // The rows of a workspace the differ re-parsed are still queued; they are + // direct entries, so they go out ahead of the rows the named pass enqueues. + manager.drain_dependency_list(); named = enqueue_named_updates( manager, &direct_deps_before, @@ -636,8 +595,16 @@ pub fn install_with_manager( if manager.pending_task_count() > 0 || manager.peer_dependencies.readable_length() > 0 || !named.latest_rows.is_empty() + || !transitive.deferred.is_empty() { - resolve_pending_tasks(manager, &root, log_level, &mut named)?; + resolve_pending_tasks( + manager, + &root, + log_level, + &mut named, + &mut transitive, + &direct_deps_before, + )?; } direct_deps_before.redirect_dependents(&mut manager.lockfile); @@ -821,6 +788,12 @@ pub fn install_with_manager( } } + if manager.options.enable.frozen_lockfile() && !manager.options.do_.save_lockfile() { + if let Some(source) = load_result.migrated_from() { + note_migrated_lockfile_not_saved(manager, source); + } + } + // BACKREF: `manager.lockfile` is a `Box` whose allocation is // never replaced for the remainder of this function (only its fields // mutate). Wrap once as `ParentRef` so the two `save_lockfile` read sites @@ -852,6 +825,11 @@ pub fn install_with_manager( break 'install_summary PackageInstallSummary::default(); } + // Every workspace is a root dependency, so without a diff the workspace set is unchanged. + if had_any_diffs { + remove_stale_workspace_links(&lockfile_before_clean, &manager.lockfile); + } + let mut linker = manager.options.node_linker; loop { match linker { @@ -932,7 +910,7 @@ pub fn install_with_manager( // It's unnecessary work to re-save the lockfile if there are no changes. // A loaded text lockfile is never re-saved just to bump its version: an // existing `bun.lock` keeps the version it was written with. - let should_save_lockfile = saves_migrated_lockfile(&load_result, save_format) + let should_save_lockfile = converts_binary_lockfile_to_text(&load_result, save_format) // check `save_lockfile` after checking if loaded from binary and save format is text // because `save_lockfile` is set to false for `--frozen-lockfile` || (manager.options.do_.save_lockfile() @@ -946,6 +924,7 @@ pub fn install_with_manager( || manager.options.enable.force_save_lockfile())); if should_save_lockfile { + super::package_json_write_back::write_migrated_root(manager); save_lockfile( manager, &load_result, @@ -964,12 +943,15 @@ pub fn install_with_manager( manager.summary.add = manager.lockfile.packages.len() as u32; } - if manager.options.do_.save_yarn_lock() { - write_yarn_lock_with_progress(manager, log_level)?; - } + if !manager.options.dry_run { + if manager.options.do_.save_yarn_lock() { + write_yarn_lock_with_progress(manager, log_level)?; + } - if manager.options.do_.run_scripts() && install_root_dependencies && !manager.options.global { - run_root_lifecycle_scripts(manager, ctx, log_level)?; + if manager.options.do_.run_scripts() && install_root_dependencies && !manager.options.global + { + run_root_lifecycle_scripts(manager, ctx, log_level)?; + } } if log_level != Options::LogLevel::Silent { @@ -1168,9 +1150,15 @@ fn print_install_summary( { // Hot no-op path (install/fastify bench): kept inline. let count = this.lockfile.packages.len() as PackageID; + // "(no changes)" would misreport an update that rewrote package.json. + let note = if this.subcommand == Subcommand::Update && this.wrote_package_json { + "up to date, package.json synced" + } else { + "no changes" + }; if count != install_summary.skipped { bun_core::pretty!( - "Checked {} install{} across {} package{} (no changes) ", + "Checked {} install{} across {} package{} ({}) ", install_summary.skipped, if install_summary.skipped == 1 { "" @@ -1179,19 +1167,21 @@ fn print_install_summary( }, count, if count == 1 { "" } else { "s" }, + note, ); Output::print_start_end_stdout(ctx.start_time, nano_timestamp()); printed_timestamp = true; print_blocked_packages_info(install_summary, this.options.global); } else { bun_core::pretty!( - "Done! Checked {} package{} (no changes) ", + "Done! Checked {} package{} ({}) ", install_summary.skipped, if install_summary.skipped == 1 { "" } else { "s" }, + note, ); Output::print_start_end_stdout(ctx.start_time, nano_timestamp()); printed_timestamp = true; @@ -1427,59 +1417,26 @@ fn overrides_field_name( return "overrides"; }; let root = entry.root; - if root.as_property(b"overrides").is_none() && root.as_property(b"resolutions").is_some() { - "resolutions" - } else { - "overrides" + match ( + root.as_property(b"overrides").is_some(), + root.as_property(b"resolutions").is_some(), + ) { + (false, true) => "resolutions", + (true, true) => "overrides or resolutions", + _ => "overrides", } } pub(crate) fn loaded_lockfile_name(load_result: &lockfile::LoadResult) -> &'static str { - if load_result.loaded_from_binary_lockfile() { + if let Some(source) = load_result.migrated_from() { + source + } else if load_result.loaded_from_binary_lockfile() { "bun.lockb" } else { "bun.lock" } } -/// Adds a contextual error for a dependency resolution failure. -/// This provides better error messages than just propagating the raw error. -/// The error is logged to manager.log, and the install will fail later when -/// manager.log.hasErrors() is checked. -#[cold] -#[inline(never)] -fn add_dependency_error(manager: &mut PackageManager, dependency: &Dependency, err: crate::Error) { - // reshaped for borrowck — capture the realname slice before - // taking `&mut` on `manager.log`. - let realname = dependency.realname(); - let path = manager.lockfile.str(&realname).to_vec(); - let path_fmt = bun_core::fmt::fmt_path( - &path, - bun_core::fmt::PathFormatOptions { - path_sep: match dependency.version.tag { - DependencyVersionTag::Folder => bun_core::fmt::PathSep::Auto, - _ => bun_core::fmt::PathSep::Any, - }, - ..Default::default() - }, - ); - - let log = manager.log_mut(); - if dependency.behavior.is_optional() || dependency.behavior.is_peer() { - log.add_warning_with_note( - None, - Default::default(), - err.name().as_bytes(), - format_args!("error occurred while resolving {}", path_fmt), - ); - } else { - log.add_zig_error_with_note( - err.name(), - format_args!("error occurred while resolving {}", path_fmt), - ); - } -} - // ─── cold install branches ──────────────────────────────────────────────── // These are the rarely-taken arms of `install_with_manager` (lockfile load // error reporting, building a brand-new lockfile, the network resolve loop, @@ -1526,7 +1483,7 @@ fn report_lockfile_load_error( Ok(()) } -/// Returns the rows the plan re-resolved so the overrides/catalogs invalidation loops that follow leave them pinned; only tracked when those loops will run. +/// Returns the rows the plan re-resolved so the overrides/catalogs invalidation pass that follows leaves them pinned; only tracked when that pass will run. fn enqueue_transitive( manager: &mut PackageManager, transitive: &TransitiveUpdate, @@ -1539,6 +1496,39 @@ fn enqueue_transitive( transitive.enqueue_tracked(manager) } +/// Re-resolves the rows `selects` in package order (root first, so later rows dedupe onto what its rows append), +/// skipping rows in no list (the root's loaded rows the differ replaced) and `pinned_rows` (the update plan's). +fn reresolve_owned_rows( + manager: &mut PackageManager, + pinned_rows: &DynamicBitSet, + selects: impl Fn(&Dependency) -> bool, +) { + // Resolving appends packages and rows; only the lists present now are walked. + let lists: Vec = + manager.lockfile.packages.items_dependencies().to_vec(); + for list in lists { + for dep_id in list.begin()..list.end() { + if pinned_rows.is_set_allow_out_of_bound(dep_id as usize, false) { + continue; + } + let dependency = manager.lockfile.buffers.dependencies[dep_id as usize].clone(); + if !selects(&dependency) { + continue; + } + manager.lockfile.buffers.resolutions[dep_id as usize] = invalid_package_id; + if let Err(err) = enqueue_dependency_with_main( + manager, + dep_id, + &dependency, + invalid_package_id, + false, + ) { + add_dependency_error(manager, &dependency, err); + } + } + } +} + #[derive(Default)] struct NamedUpdates { /// Invalidated rows paired with the package they resolved to, for redirect_moved_edges. @@ -1563,11 +1553,12 @@ fn enqueue_named_updates( let mut matched_elsewhere = DynamicBitSet::init_empty(requests).unwrap_or_oom(); let mut named = NamedUpdates::default(); let mut peer_rows: Vec = Vec::new(); + let mut rows: Vec<(DependencyID, PackageID)> = Vec::new(); let dependencies_len = manager.lockfile.buffers.dependencies.len(); for dependency_i in 0..dependencies_len { - let dependency = manager.lockfile.buffers.dependencies[dependency_i].clone(); + let dependency = &manager.lockfile.buffers.dependencies[dependency_i]; let package_id = manager.lockfile.buffers.resolutions[dependency_i]; - let Some(request) = index_of_named_update(manager, &dependency, package_id) else { + let Some(request) = index_of_named_update(manager, dependency, package_id) else { continue; }; if !walkable.is_set(dependency_i) { @@ -1587,11 +1578,21 @@ fn enqueue_named_updates( } continue; } - manager.lockfile.buffers.resolutions[dependency_i] = invalid_package_id; - named.moved.push((dependency_i as DependencyID, package_id)); + rows.push((dependency_i as DependencyID, package_id)); + } + + // Direct rows resolve first, so rows regular packages own can land on the copy they move to + // (`get_or_put_resolved_package_with_find_result`) and redirected edges follow the direct entry. + let (direct_rows, transitive_rows): (Vec<_>, Vec<_>) = rows + .into_iter() + .partition(|&(dependency_i, _)| manager.lockfile.is_workspace_dependency(dependency_i)); + for (dependency_i, package_id) in direct_rows.into_iter().chain(transitive_rows) { + let dependency = manager.lockfile.buffers.dependencies[dependency_i as usize].clone(); + manager.lockfile.buffers.resolutions[dependency_i as usize] = invalid_package_id; + named.moved.push((dependency_i, package_id)); if let Err(err) = enqueue_dependency_with_main( manager, - dependency_i as DependencyID, + dependency_i, &dependency, invalid_package_id, false, @@ -1829,24 +1830,13 @@ fn record_updating_package_versions(manager: &mut PackageManager) { if original_resolution.tag != ResolutionTag::Npm { continue; } - - let mut original = original_resolution.npm().version; - let tag_total = original.tag.pre.len() + original.tag.build.len(); - if tag_total > 0 { - let mut tag_buf = vec![0u8; tag_total].into_boxed_slice(); - let mut ptr = &mut tag_buf[..]; - original.tag = original_resolution - .npm() - .version - .tag - .clone_into(&lockfile.buffers.string_bytes, &mut ptr); - - entry_ptr.original_version_string_buf = tag_buf; - } - - entry_ptr.original_version = Some(original); + entry_ptr.set_original_version( + original_resolution.npm().version, + &lockfile.buffers.string_bytes, + ); } } + PackageJSONEditor::record_catalog_originals(manager).unwrap_or_oom(); } fn root_package_json_source( @@ -1912,6 +1902,11 @@ fn create_new_lockfile_and_enqueue( Global::crash(); } + // A loaded lockfile already describes the project; a migrated pnpm-lock.yaml imports the yaml itself. + if !matches!(load_result, lockfile::LoadResult::Ok { .. }) { + crate::pnpm::migrate_pnpm_workspace_config(manager)?; + } + let source_copy = root_package_json_source(manager, root_package_json_path)?; let mut resolver: () = (); @@ -1961,6 +1956,8 @@ fn resolve_pending_tasks( root: &lockfile::Package, log_level: Options::LogLevel, named: &mut NamedUpdates, + transitive: &mut TransitiveUpdate, + direct_deps_before: &DirectDependencies, ) -> crate::Result<()> { if root.dependencies.len > 0 { let _ = manager.get_cache_directory(); @@ -1980,6 +1977,10 @@ fn resolve_pending_tasks( wait_for_resolution(manager)?; + if transitive.plan_unanchored(manager, direct_deps_before)? { + wait_for_resolution(manager)?; + } + if !named.latest_rows.is_empty() { let child_moves = refresh_children_of_named(manager, &named.latest_rows)?; named.moved.extend(child_moves); @@ -2021,7 +2022,7 @@ fn refresh_children_of_named( }) .collect() }; - index_sort::sort_indices_unstable(&mut package_ids, &mut |a, b| a.cmp(&b)); + package_ids.sort_unstable(); package_ids.dedup(); if package_ids.is_empty() { return Ok(Vec::new()); @@ -2035,6 +2036,7 @@ fn wait_for_resolution(manager: &mut PackageManager) -> crate::Result<()> { if manager.pending_task_count() > 0 { wait_for_everything_except_peers(manager)?; } + manager.lockfile.mark_settled_packages(); // Resolving a peer dep can create a NEW package whose own peer deps // get re-queued to `peer_dependencies` during `drainDependencyList`. @@ -2146,8 +2148,9 @@ fn run_security_scanner( } } -// bun.lockb / package-lock.json / yarn.lock / pnpm-lock.yaml -> bun.lock is written even under --frozen-lockfile. -fn saves_migrated_lockfile( +// bun.lockb -> bun.lock (`--save-text-lockfile --frozen-lockfile --lockfile-only`) is the one write while +// `Do::SAVE_LOCKFILE` is off; migrations (which may report `Format::Binary` too) save via `FORCE_SAVE_LOCKFILE`. +fn converts_binary_lockfile_to_text( load_result: &lockfile::LoadResult, save_format: lockfile::Format, ) -> bool { @@ -2155,10 +2158,30 @@ fn saves_migrated_lockfile( && matches!( load_result, lockfile::LoadResult::Ok(ok) - if ok.format == lockfile::Format::Binary || ok.migrated != lockfile::Migrated::None + if ok.format == lockfile::Format::Binary && ok.migrated == lockfile::Migrated::None ) } +#[cold] +#[inline(never)] +fn note_migrated_lockfile_not_saved(manager: &PackageManager, source: &str) { + if manager.options.log_level.is_silent() { + return; + } + Output::flush(); + let files = if manager.migrated_package_json_moves.is_empty() { + "bun.lock" + } else { + "bun.lock and package.json" + }; + bun_core::note!( + "the lockfile is frozen, so the migration from {} was not written to {}; run 'bun install' and commit the result", + source, + files, + ); + Output::flush(); +} + #[cold] #[inline(never)] #[allow(clippy::too_many_arguments)] @@ -2172,8 +2195,8 @@ fn save_lockfile_only( packages_len_before_install: usize, log_level: Options::LogLevel, ) -> crate::Result<()> { - if (manager.options.enable.frozen_lockfile() - && !saves_migrated_lockfile(load_result, save_format)) + if (!manager.options.do_.save_lockfile() + && !converts_binary_lockfile_to_text(load_result, save_format)) || (manager.subcommand == Subcommand::Dedupe && manager.dedupe_report.is_none()) { Output::flush(); @@ -2186,6 +2209,7 @@ fn save_lockfile_only( packages_len_before_install, )?; + super::package_json_write_back::write_migrated_root(manager); let saved = save_lockfile( manager, load_result, @@ -2195,6 +2219,7 @@ fn save_lockfile_only( packages_len_before_install, log_level, )?; + super::package_json_write_back::flush(manager)?; if manager.subcommand == Subcommand::Dedupe { if manager.options.do_.summary() { diff --git a/src/install/PackageManager/package_json_write_back.rs b/src/install/PackageManager/package_json_write_back.rs index becf4d246f84..492903e9eb8f 100644 --- a/src/install/PackageManager/package_json_write_back.rs +++ b/src/install/PackageManager/package_json_write_back.rs @@ -10,7 +10,7 @@ use crate::dependency::DependencyExt as _; use crate::lockfile::package::PackageColumns as _; use crate::lockfile::{Lockfile, Package}; use crate::resolution::Tag as ResolutionTag; -use crate::{Dependency, PackageID, PackageNameHash, invalid_package_id}; +use crate::{Dependency, Features, PackageID, PackageNameHash, invalid_package_id}; use super::add_catalog; use super::add_remove_with_filter::{ @@ -58,6 +58,25 @@ fn root_target() -> WorkspaceTarget { } } +/// Writes the root package.json a pnpm migration edited in memory, just before the migrated lockfile is saved: +/// a package.json that cannot take the moved fields fails the command before a lockfile that depends on them exists. +/// Only the places that save the lockfile call this, so a load that is never saved leaves the file alone. +pub fn write_migrated_root(manager: &mut PackageManager) { + if manager.migrated_package_json_moves.is_empty() { + return; + } + let moved = core::mem::take(&mut manager.migrated_package_json_moves); + if !write_target(manager, &root_target()) { + Global::exit(1); + } + if !manager.options.log_level.is_silent() { + bun_core::pretty_errorln!( + "copied {} in package.json", + moved.join(", ") + ); + } +} + /// Phase 1 (before bun.lock is saved): write the resolved versions into the edited package.json entries and re-derive bun.lock's declared columns from them. #[inline] pub(crate) fn edit_after_resolve(manager: &mut PackageManager) -> crate::Result<()> { @@ -267,7 +286,7 @@ fn target_package_ids(lockfile: &Lockfile, edited: &[EditedPackageJson]) -> Vec< /// Re-parses the edited files the way `bun install` would and copies every declared literal that differs (and, for the root, `overrides` + `catalogs`) into `manager.lockfile`, so the next install's differ sees no change. fn sync_lockfile(manager: &mut PackageManager, edited: &[EditedPackageJson]) -> crate::Result<()> { let mut scratch = super::workspace_manifests::ScratchManifests::new(); - scratch.parse_root(manager)?; + scratch.parse_root(manager, Features::main())?; let mut root_pkg = Some(core::mem::take(&mut scratch.root)); let mut parsed: Vec<(usize, Package)> = Vec::with_capacity(edited.len()); for (i, e) in edited.iter().enumerate() { @@ -406,7 +425,11 @@ pub(crate) fn flush(manager: &mut PackageManager) -> Result<(), crate::Error> { if unchanged_on_disk(manager, &e.target) { continue; } - any_failed |= !write_target(manager, &e.target); + if write_target(manager, &e.target) { + manager.wrote_package_json = true; + } else { + any_failed = true; + } } if any_failed { Global::exit(1); diff --git a/src/install/PackageManager/patchPackage.rs b/src/install/PackageManager/patchPackage.rs index 49029f7cd401..dae36ff83267 100644 --- a/src/install/PackageManager/patchPackage.rs +++ b/src/install/PackageManager/patchPackage.rs @@ -270,8 +270,14 @@ pub fn do_patch_commit( // `compute_cache_dir_and_subpath` resolves `pkg.resolution`'s strings against `manager.lockfile`. manager.lockfile = lockfile; let name = manager.lockfile.str(&pkg.name).to_vec(); - let cache_result = - compute_cache_dir_and_subpath(manager, &name, &pkg.resolution, &mut folder_path_buf, None); + let cache_result = compute_cache_dir_and_subpath( + manager, + &name, + &pkg.resolution, + &pkg.meta.integrity, + &mut folder_path_buf, + None, + ); let cache_dir: Fd = cache_result.cache_dir; let cache_dir_subpath: &ZStr = cache_result.cache_dir_subpath; let changes_dir: &[u8] = &changes_dir; @@ -344,6 +350,7 @@ pub fn do_patch_commit( random_tempdir.as_bytes(), sys::RenameOptions { move_fallback: true, + ..Default::default() }, ) .is_err() @@ -399,6 +406,7 @@ pub fn do_patch_commit( patch_tag_tmpname.as_bytes(), sys::RenameOptions { move_fallback: true, + ..Default::default() }, ) { bun_core::warn!( @@ -431,7 +439,7 @@ pub fn do_patch_commit( random_tempdir.as_bytes(), new_folder_handle.fd, b"node_modules", - sys::RenameOptions { move_fallback: true }, + sys::RenameOptions { move_fallback: true, ..Default::default() }, ) { bun_core::warn!("failed renaming nested node_modules folder, this may cause issues: {}", e); } @@ -443,7 +451,7 @@ pub fn do_patch_commit( patch_tag_tmpname.as_bytes(), new_folder_handle.fd, patch_tag, - sys::RenameOptions { move_fallback: true }, + sys::RenameOptions { move_fallback: true, ..Default::default() }, ) { bun_core::warn!("failed renaming the bun patch tag, this may cause issues: {}", e); } @@ -588,6 +596,7 @@ pub fn do_patch_commit( path_in_patches_dir, sys::RenameOptions { move_fallback: true, + ..Default::default() }, ) { Output::err(e, "failed renaming patch file to patches dir", ()); @@ -858,6 +867,7 @@ pub fn prepare_patch(manager: &mut PackageManager) -> Result<(), crate::Error> { manager, &name, &actual_package.resolution, + &actual_package.meta.integrity, &mut folder_path_buf, existing_patchfile_hash, ); @@ -914,10 +924,12 @@ pub fn prepare_patch(manager: &mut PackageManager) -> Result<(), crate::Error> { }; let pkg_resolution = pkg.resolution; + let pkg_integrity = pkg.meta.integrity; let cache_result = compute_cache_dir_and_subpath( manager, &pkg_name, &pkg_resolution, + &pkg_integrity, &mut folder_path_buf, existing_patchfile_hash, ); @@ -1396,7 +1408,7 @@ fn pkg_info_for_name_and_version( bun_core::pretty_error!( " {}@{}\n", bstr::BStr::new(pkg.name.slice(strbuf)), - pkg.resolution.fmt(strbuf, PathSep::Posix) + bun_fmt::EscapeControlChars(pkg.resolution.fmt(strbuf, PathSep::Posix)) ); if i + 1 < pairs.len() { diff --git a/src/install/PackageManager/processDependencyList.rs b/src/install/PackageManager/processDependencyList.rs index 7fdd1f654dac..e3fca237d978 100644 --- a/src/install/PackageManager/processDependencyList.rs +++ b/src/install/PackageManager/processDependencyList.rs @@ -15,11 +15,11 @@ use crate::package_manager_real::options::LogLevel; use crate::package_manager_real::{ PackageManager, TaskCallbackList, enqueue, resolution as pm_resolution, }; -use crate::repository_real::{Repository, RepositoryExt as _}; +use crate::repository_real::RepositoryExt as _; use crate::resolution::{ResolutionType, Tag as ResolutionTag, TaggedValue}; use crate::{ - DependencyID, ExtractData, Features, INVALID_PACKAGE_ID, PackageID, Resolution, - TaskCallbackContext, initialize_store, + ExtractData, Features, INVALID_PACKAGE_ID, PackageID, Resolution, TaskCallbackContext, + dependency, initialize_store, }; // ────────────────────────────────────────────────────────────────────────── @@ -29,16 +29,12 @@ use crate::{ pub struct GitResolver<'a> { pub(crate) resolved: &'a [u8], pub(crate) resolution: &'a Resolution, - pub(crate) dep_id: DependencyID, - /// Owned scratch buffer that - /// `Package::parse_with_json` may assign when the package.json `name` - /// field is missing (see `ResolverContext::set_new_name`). - pub(crate) new_name: Vec, + /// `Repository::fallback_package_name`, copied out before parsing starts + /// because it is sliced from the lockfile string buffer parsing appends to. + pub(crate) fallback_name: &'a [u8], } impl<'a> ResolverContext for GitResolver<'a> { - const IS_GIT_RESOLVER: bool = true; - fn check_bundled_dependencies() -> bool { true } @@ -67,20 +63,8 @@ impl<'a> ResolverContext for GitResolver<'a> { })) } - fn resolution(&self) -> &Resolution { - self.resolution - } - fn dep_id(&self) -> DependencyID { - self.dep_id - } - fn new_name(&self) -> &[u8] { - &self.new_name - } - fn set_new_name(&mut self, name: Vec) { - self.new_name = name; - } - fn take_new_name(&mut self) -> Vec { - core::mem::take(&mut self.new_name) + fn fallback_name(&self) -> Option> { + Some(self.fallback_name.to_vec()) } } @@ -117,6 +101,10 @@ impl<'a> ResolverContext for TarballResolver<'a> { _ => unreachable!(), })) } + + fn fallback_name(&self) -> Option> { + Some(dependency::fallback_package_name(self.url).to_vec()) + } } // ────────────────────────────────────────────────────────────────────────── @@ -128,7 +116,6 @@ impl PackageManager { pub(crate) fn process_extracted_tarball_package( &mut self, package_id: &mut PackageID, - dep_id: DependencyID, resolution: &Resolution, data: &ExtractData, log_level: LogLevel, @@ -136,11 +123,14 @@ impl PackageManager { match resolution.tag { ResolutionTag::Git | ResolutionTag::Github => { let mut package = 'package: { + let fallback_name: Vec = resolution + .repository() + .fallback_package_name(self.lockfile.buffers.string_bytes.as_slice()) + .to_vec(); let mut resolver = GitResolver { resolved: &data.resolved, resolution, - dep_id, - new_name: Vec::new(), + fallback_name: &fallback_name, }; let mut pkg = Package::default(); @@ -164,10 +154,13 @@ impl PackageManager { Output::err( err, "failed to parse package.json for {}", - format_args!("{}", resolution.fmt_url(string_buf)), + format_args!( + "{}", + bun_core::fmt::for_terminal(resolution.fmt_url(string_buf)) + ), ); } - Global::crash(); + self.crash(); } let has_scripts = pkg.scripts.has_any() || { @@ -184,25 +177,15 @@ impl PackageManager { } // package.json doesn't exist, no dependencies to worry about but we need to decide on a name for the dependency - // tag is `.git` or `.github`; both store `Repository`. - let repo = *resolution.repository(); - - let new_name = Repository::create_dependency_name_from_version_literal( - &repo, - self.lockfile.buffers.string_bytes.as_slice(), - &self.lockfile.buffers.dependencies[dep_id as usize], - ); - // `defer manager.allocator.free(new_name)` — `new_name: Vec` drops at scope end. - { let mut builder = self.lockfile.string_builder(); - builder.count(&new_name); + builder.count(&fallback_name); resolver.count(&mut builder, &Expr::default()); bun_core::handle_oom(builder.allocate()); - let name = builder.append::(&new_name); + let name = builder.append::(&fallback_name); pkg.name = name.value; pkg.name_hash = name.hash; @@ -260,11 +243,11 @@ impl PackageManager { let string_buf = self.lockfile.buffers.string_bytes.as_slice(); bun_core::pretty_errorln!( "error: expected package.json in {} to be a JSON file: {}\n", - resolution.fmt_url(string_buf), + bun_core::fmt::for_terminal(resolution.fmt_url(string_buf)), err.name(), ); } - Global::crash(); + self.crash(); } let has_scripts = package.scripts.has_any() || { @@ -312,7 +295,7 @@ impl PackageManager { let string_buf = self.lockfile.buffers.string_bytes.as_slice(); bun_core::pretty_errorln!( "error: expected package.json in {} to be a JSON file: {}\n", - resolution.fmt_url(string_buf), + bun_core::fmt::for_terminal(resolution.fmt_url(string_buf)), err.name(), ); } @@ -384,7 +367,6 @@ impl PackageManager { resolution, install_peer, pm_resolution::assign_root_resolution, - Some(PackageManager::fail_root_resolution), true, )?; if let Some(ptr) = any_root { diff --git a/src/install/PackageManager/remove_stale_workspace_links.rs b/src/install/PackageManager/remove_stale_workspace_links.rs new file mode 100644 index 000000000000..8fc0b0e1ce18 --- /dev/null +++ b/src/install/PackageManager/remove_stale_workspace_links.rs @@ -0,0 +1,54 @@ +use bstr::BStr; +use bun_paths::AutoAbsPathChecked; + +use crate::isolated_install::installer::remove_link; +use crate::lockfile::Lockfile; +use crate::lockfile::package::PackageColumns as _; +use crate::package_installer::alias_is_safe_install_target; +use crate::package_manager_real::PackageManager; +use crate::resolution::Tag as ResolutionTag; + +/// Linkers only visit names in the new lockfile; a workspace name that left it is unlinked here. +pub(crate) fn remove_stale_workspace_links(previous: &Lockfile, current: &Lockfile) { + let previous_packages = previous.packages.slice(); + let resolutions = previous_packages.items_resolution(); + let name_hashes = previous_packages.items_name_hash(); + + for (pkg_id, name) in previous_packages.items_name().iter().enumerate() { + let name = name.slice(&previous.buffers.string_bytes); + if resolutions[pkg_id].tag != ResolutionTag::Workspace + || current.workspace_paths.contains_key(&name_hashes[pkg_id]) + || !alias_is_safe_install_target(name) + { + continue; + } + + remove_link_in(b"", name); + for workspace_path in current.workspace_paths.values() { + remove_link_in(workspace_path.slice(¤t.buffers.string_bytes), name); + } + } +} + +/// `readlink` is the symlink (or junction) check; a real directory or file there is left alone. +fn remove_link_in(package_dir: &[u8], name: &[u8]) { + let mut path = AutoAbsPathChecked::init_top_level_dir(); + if path.append(package_dir).is_err() + || path.append(b"node_modules").is_err() + || path.append(name).is_err() + { + return; + } + + let mut link_target = bun_paths::path_buffer_pool::get(); + if bun_sys::readlink(path.slice_z(), &mut link_target).is_err() { + return; + } + + bun_output::scoped_log!( + PackageManager, + "removing stale workspace link {}", + BStr::new(path.slice()) + ); + let _ = remove_link(path.slice_z()); +} diff --git a/src/install/PackageManager/runTasks.rs b/src/install/PackageManager/runTasks.rs index 05c3c87d1002..cc4f6aef3fdd 100644 --- a/src/install/PackageManager/runTasks.rs +++ b/src/install/PackageManager/runTasks.rs @@ -27,14 +27,13 @@ use crate::isolated_install::installer as store_installer; use crate::isolated_install::store::{EntryColumns as _, NodeColumns as _}; use crate::lifecycle_script_runner::InstallCtx; use crate::network_task::{Authorization, ForTarballError}; -use crate::package_manifest_map::Value as ManifestEntry; use bun_core::fmt::PathSep; use bun_install::lockfile::Package; use bun_install::package_manager_task as Task; // Import the *module* under the `Options` name so `Options::LogLevel` resolves as a path // (matches the `Task` module-alias pattern above and `CommandLineArguments.rs`). use super::package_manager_options as Options; -use super::package_manager_options::{Do, Enable}; +use super::package_manager_options::{Do, Enable, RequestKind}; use crate::isolated_install::store as Store; // ────────────────────────────────────────────────────────────────────────── @@ -110,7 +109,11 @@ pub trait RunTasksCallbacks { ) { unreachable!() } - fn on_extract_store_installer(_ctx: &mut Self::Ctx, _task_id: Task::Id) { + fn on_extract_store_installer( + _ctx: &mut Self::Ctx, + _task_id: Task::Id, + _data: &bun_install::ExtractData, + ) { unreachable!() } @@ -413,6 +416,45 @@ pub fn run_tasks( } } + let request_failed = task + .response + .metadata + .as_ref() + .is_none_or(|metadata| metadata.response.status_code > 399); + // Manifests-only callers (migrations, `bun outdated`) have nothing waiting that could fall back. + if request_failed + && is_extended_manifest + && !C::MANIFESTS_ONLY + && fall_back_to_abbreviated_manifest(manager, task.task_id) + { + let reason = match task.response.metadata.as_ref() { + Some(metadata) => format!("HTTP {}", metadata.response.status_code), + None => task + .response + .fail + .map(crate::Error::from) + .unwrap_or(crate::Error::HTTPError) + .name() + .to_string(), + }; + bun_ast::add_warning_pretty!( + manager.log_mut(), + None, + bun_ast::Loc::EMPTY, + "{} downloading the full package metadata for {}, using the abbreviated metadata: its libc field will not be checked", + reason, + bstr::BStr::new(name), + ); + + process_manifest_task_queue::( + manager, + task.task_id, + extract_ctx, + install_peer, + )?; + continue; + } + let Some(metadata) = task.response.metadata.as_ref() else { // Handle non-retry-able errors. let err = task @@ -478,23 +520,34 @@ pub fn run_tasks( continue; } + // What to configure, on 401/403. + let note = match response.status_code { + 401 | 403 => manager.options.missing_credentials_note( + name, + &task.url_buf, + RequestKind::Manifest, + ), + _ => Vec::new(), + }; if manager.is_network_task_required(task.task_id) { bun_ast::add_error_pretty!( manager.log_mut(), None, bun_ast::Loc::EMPTY, - "GET {} - {}", - bstr::BStr::new(metadata.url.slice()), + "GET {} - {}{}", + bun_core::fmt::redacted_npm_url(metadata.url.slice()), response.status_code, + bstr::BStr::new(¬e), ); } else { bun_ast::add_warning_pretty!( manager.log_mut(), None, bun_ast::Loc::EMPTY, - "GET {} - {}", - bstr::BStr::new(metadata.url.slice()), + "GET {} - {}{}", + bun_core::fmt::redacted_npm_url(metadata.url.slice()), response.status_code, + bstr::BStr::new(¬e), ); } if manager.subcommand != Subcommand::Remove { @@ -542,15 +595,9 @@ pub fn run_tasks( manifest.pkg.public_max_age = timestamp_this_tick.unwrap(); - // reshaped for borrowck — - // `bun_collections::HashMap` lacks `get_or_put` for - // non-`Default` values, so insert by-value (overwriting - // any prior entry) and reborrow. + // `insert` may keep an extended manifest that arrived meanwhile; that one is written back. let name_hash = manifest.pkg.name.hash; - manager - .manifests - .hash_map - .insert(name_hash, ManifestEntry::Manifest(manifest)); + manager.manifests.insert(name_hash, manifest)?; if manager.options.enable.contains(Enable::MANIFEST_CACHE) { // reshaped for borrowck — compute the @@ -579,16 +626,9 @@ pub fn run_tasks( continue; } - let dependency_list_entry = manager - .task_queue - .get_mut(&task.task_id) - .expect("infallible: task queued"); - - let dependency_list = core::mem::take(dependency_list_entry); - - process_dependency_list_for_ctx::( + process_manifest_task_queue::( manager, - dependency_list, + task.task_id, extract_ctx, install_peer, )?; @@ -674,12 +714,14 @@ pub fn run_tasks( manager.log_mut(), None, bun_ast::Loc::EMPTY, - "warn: {} downloading tarball {}@{}. Retrying {}/{}...", + "{} downloading tarball {}@{}. Retrying {}/{}...", bstr::BStr::new(err.name().as_bytes()), - bstr::BStr::new(extract.name.slice()), - extract - .resolution - .fmt(&manager.lockfile.buffers.string_bytes, PathSep::Auto,), + bun_core::fmt::escape_control_chars(extract.name.slice()), + bun_core::fmt::for_terminal( + extract + .resolution + .fmt(&manager.lockfile.buffers.string_bytes, PathSep::Auto), + ), task.retried, manager.options.max_retry_count, ); @@ -746,10 +788,12 @@ pub fn run_tasks( bun_ast::Loc::EMPTY, "{} downloading tarball {}@{}", err.name(), - bstr::BStr::new(extract.name.slice()), - extract - .resolution - .fmt(&manager.lockfile.buffers.string_bytes, PathSep::Auto,), + bun_core::fmt::escape_control_chars(extract.name.slice()), + bun_core::fmt::for_terminal( + extract + .resolution + .fmt(&manager.lockfile.buffers.string_bytes, PathSep::Auto), + ), ); } else { bun_ast::add_warning_pretty!( @@ -758,10 +802,12 @@ pub fn run_tasks( bun_ast::Loc::EMPTY, "{} downloading tarball {}@{}", err.name(), - bstr::BStr::new(extract.name.slice()), - extract - .resolution - .fmt(&manager.lockfile.buffers.string_bytes, PathSep::Auto,), + bun_core::fmt::escape_control_chars(extract.name.slice()), + bun_core::fmt::for_terminal( + extract + .resolution + .fmt(&manager.lockfile.buffers.string_bytes, PathSep::Auto), + ), ); } if manager.subcommand != Subcommand::Remove { @@ -828,23 +874,37 @@ pub fn run_tasks( continue; } + let note = match response.status_code { + 401 | 403 => manager.options.missing_credentials_note( + extract.name.slice(), + &task.url_buf, + RequestKind::Tarball(task.authorization), + ), + _ => Vec::new(), + }; if is_required { bun_ast::add_error_pretty!( manager.log_mut(), None, bun_ast::Loc::EMPTY, - "GET {} - {}", - bstr::BStr::new(metadata.url.slice()), + "GET {} - {}{}", + bun_core::fmt::EscapeControlChars(bun_core::fmt::redacted_npm_url( + metadata.url.slice() + )), response.status_code, + bstr::BStr::new(¬e), ); } else { bun_ast::add_warning_pretty!( manager.log_mut(), None, bun_ast::Loc::EMPTY, - "GET {} - {}", - bstr::BStr::new(metadata.url.slice()), + "GET {} - {}{}", + bun_core::fmt::EscapeControlChars(bun_core::fmt::redacted_npm_url( + metadata.url.slice() + )), response.status_code, + bstr::BStr::new(¬e), ); } if manager.subcommand != Subcommand::Remove { @@ -1003,18 +1063,7 @@ pub fn run_tasks( continue; } - let dependency_list_entry = manager - .task_queue - .get_mut(&task.id) - .expect("infallible: task queued"); - let dependency_list = core::mem::take(dependency_list_entry); - - process_dependency_list_for_ctx::( - manager, - dependency_list, - extract_ctx, - install_peer, - )?; + process_manifest_task_queue::(manager, task.id, extract_ctx, install_peer)?; if let Some(name) = progress_name { manager.set_node_name::( @@ -1146,13 +1195,12 @@ pub fn run_tasks( log_level, ); } else if C::IS_STORE_INSTALLER { - C::on_extract_store_installer(extract_ctx, task.id); + C::on_extract_store_installer(extract_ctx, task.id, task.data_extract()); } else { unreachable!("unexpected context type"); } } else if let Some(pkg) = manager.process_extracted_tarball_package( &mut package_id, - dependency_id, resolution, // Tag-checked accessor (debug_asserts Extract|LocalTarball); // shared `&task` here coexists with the field-disjoint @@ -1475,13 +1523,16 @@ pub fn run_tasks( log_level, ); } else if C::IS_STORE_INSTALLER { - C::on_extract_store_installer(extract_ctx, task.id); + C::on_extract_store_installer( + extract_ctx, + task.id, + task.data_git_checkout(), + ); } else { unreachable!("unexpected context type"); } } else if let Some(pkg) = manager.process_extracted_tarball_package( &mut package_id, - git_checkout.dependency_id, resolution, // Tag-checked accessor (debug_asserts GitCheckout); shared // `&task` here coexists with the field-disjoint @@ -1744,6 +1795,73 @@ pub fn has_created_network_task( gpe.found_existing } +/// `has_created_network_task` per manifest document (abbreviated or extended, see `needs_extended_manifest`); +/// both responses drain the same `task_queue` entry and `PackageManifestMap::insert` keeps the extended one. +pub fn has_created_manifest_task( + this: &mut PackageManager, + task_id: Task::Id, + is_required: bool, + needs_extended_manifest: bool, +) -> bool { + has_created_network_task(this, task_id, is_required); + let entry = this + .network_dedupe_map + .get_mut(&task_id) + .expect("inserted by has_created_network_task"); + core::mem::replace(entry.manifest_requested(needs_extended_manifest), true) +} + +/// The caller of `has_created_manifest_task` resolved from the cache instead of sending the request; +/// the entry stays while it records anything else (the other kind in flight, a failed extended request). +pub fn manifest_request_not_sent( + this: &mut PackageManager, + task_id: Task::Id, + needs_extended_manifest: bool, +) { + let Some(entry) = this.network_dedupe_map.get_mut(&task_id) else { + return; + }; + *entry.manifest_requested(needs_extended_manifest) = false; + if !entry.is_extended_manifest + && !entry.has_abbreviated_manifest_request + && !entry.extended_manifest_failed + { + let _ = this.network_dedupe_map.remove(&task_id); + } +} + +/// `Options::needs_extended_manifest`, unless the extended request failed and the abbreviated document +/// stands in (`fall_back_to_abbreviated_manifest`); every lookup on the resolve path goes through this. +pub fn needs_extended_manifest( + this: &PackageManager, + dependency: Behavior, + task_id: Task::Id, +) -> bool { + if this.options.needs_extended_manifest_to_pick_versions() { + return true; + } + this.options.needs_extended_manifest(dependency) + && !this + .network_dedupe_map + .get(&task_id) + .is_some_and(|entry| entry.extended_manifest_failed) +} + +/// The extended request failed for good: dependencies on the package resolve from the abbreviated document +/// from now on (losing its `libc` filtering); the caller re-enqueues the waiting ones. False when versions are +/// picked from the extended document (`minimumReleaseAge` needs its publish times). +pub fn fall_back_to_abbreviated_manifest(this: &mut PackageManager, task_id: Task::Id) -> bool { + if this.options.needs_extended_manifest_to_pick_versions() { + return false; + } + let entry = this + .network_dedupe_map + .get_or_put(task_id) + .expect("unreachable"); + entry.value_ptr.extended_manifest_failed = true; + true +} + pub fn is_network_task_required(this: &PackageManager, task_id: Task::Id) -> bool { match this.network_dedupe_map.get(&task_id) { Some(v) => v.is_required, @@ -1822,7 +1940,7 @@ pub fn generate_network_task_for_tarball<'a>( // Full struct overwrite that resets // every other field (`retried`, `response`, `streaming_committed`, // `tarball_stream`, `streaming_extract_task`, `next`, `url_buf`, - // `signal_store`) to its struct default. The slot may be uninitialized + // `http_proxy_buf`, `signal_store`) to its struct default. The slot may be uninitialized // (`HiveArrayFallback::get()` heap fallback) or stale (reused hive slot). // SAFETY: `net_ptr` is the unique handle to a freshly-vended pool slot; no // other alias exists until we return it. @@ -1953,6 +2071,21 @@ impl PackageManager { } } +/// Resolves the rows waiting for this manifest; one that needs the extended document re-queues itself on the abbreviated one. +fn process_manifest_task_queue( + manager: &mut PackageManager, + task_id: Task::Id, + extract_ctx: &mut C::Ctx, + install_peer: bool, +) -> crate::Result<()> { + let dependency_list_entry = manager + .task_queue + .get_mut(&task_id) + .expect("infallible: task queued"); + let dependency_list = core::mem::take(dependency_list_entry); + process_dependency_list_for_ctx::(manager, dependency_list, extract_ctx, install_peer) +} + /// Adapter wrapping the existing `PackageManager::process_dependency_list` so /// it can be driven by a `RunTasksCallbacks` impl, dispatching `on_resolve` /// if any root dep changed. diff --git a/src/install/PackageManager/updatePackageJSONAndInstall.rs b/src/install/PackageManager/updatePackageJSONAndInstall.rs index cb75ce464347..323c2b6a181d 100644 --- a/src/install/PackageManager/updatePackageJSONAndInstall.rs +++ b/src/install/PackageManager/updatePackageJSONAndInstall.rs @@ -284,9 +284,18 @@ fn update_package_json_and_install_with_manager_with_updates( add_catalog::prepare(manager, &updates); + // Loads the lockfile, and migrating one rewrites the cached root package.json: do it before reading the entry. + let patch_commit: Option = + if matches!(manager.options.patch_features, PatchFeatures::Commit { .. }) { + let mut pathbuf = PathBuffer::uninit(); + patch_package::do_patch_commit(manager, &mut pathbuf, log_level)? + } else { + None + }; + // reshaped for borrowck — `get_with_path` returns `&mut MapEntry` // borrowed from `manager.workspace_package_json_cache`, but we then need - // `&mut *manager` for `PackageJSONEditor::edit` / `do_patch_commit` while still + // `&mut *manager` for `PackageJSONEditor::edit` while still // holding the entry. Demote to `*mut MapEntry` and re- // borrow at point of use. The cache map is not mutated again until the // next `get_with_path` call below, so the pointer remains valid. @@ -326,8 +335,7 @@ fn update_package_json_and_install_with_manager_with_updates( }; // SAFETY: see note above — pointer into `manager.workspace_package_json_cache`, // valid until the next `get_with_path`. No `&mut manager.workspace_package_json_cache` - // is taken across this borrow; `PackageJSONEditor` and `do_patch_commit` touch only - // disjoint manager fields. + // is taken across this borrow; `PackageJSONEditor` touches only disjoint manager fields. let current_package_json: &mut MapEntry = unsafe { &mut *current_package_json_ptr }; let mut current_package_json_root: bun_ast::Expr = current_package_json.root; let current_package_json_indent = current_package_json.indentation; @@ -425,23 +433,17 @@ fn update_package_json_and_install_with_manager_with_updates( } } _ => { - if matches!(manager.options.patch_features, PatchFeatures::Commit { .. }) { - let mut pathbuf = PathBuffer::uninit(); - if let Some(stuff) = - patch_package::do_patch_commit(manager, &mut pathbuf, log_level)? - { - // we're inside a workspace package, we need to edit the - // root json, not the `current_package_json` - if stuff.not_in_workspace_root { - not_in_workspace_root = Some(stuff); - } else { - PackageJSONEditor::edit_patched_dependencies( - manager, - &mut current_package_json_root, - &stuff.patch_key, - &stuff.patchfile_path, - )?; - } + if let Some(stuff) = patch_commit { + // Inside a workspace package the root package.json is edited below, not `current_package_json`. + if stuff.not_in_workspace_root { + not_in_workspace_root = Some(stuff); + } else { + PackageJSONEditor::edit_patched_dependencies( + manager, + &mut current_package_json_root, + &stuff.patch_key, + &stuff.patchfile_path, + )?; } } } @@ -483,12 +485,12 @@ fn update_package_json_and_install_with_manager_with_updates( // The Smarter™ approach is you resolve ahead of time and write to disk once! // But, turns out that's slower in any case where more than one package has to be resolved (most of the time!) // Concurrent network requests are faster than doing one and then waiting until the next batch - let new_package_json_source: Vec = package_json_writer - .ctx - .written_without_trailing_zero() - .to_vec(); - // The cache entry (`Cow<'static, [u8]>`) outlives this stack frame, so it needs its own copy. - current_package_json.source.contents = Cow::Owned(new_package_json_source.clone()); + current_package_json.source.contents = Cow::Owned( + package_json_writer + .ctx + .written_without_trailing_zero() + .to_vec(), + ); // The edits above went into a promoted copy // (`current_package_json_root`), so re-parse the // printed source so the cached AST (consumed by `FolderResolver` for workspace @@ -661,40 +663,34 @@ fn update_package_json_and_install_with_manager_with_updates( } if manager.options.do_.contains(Do::WRITE_PACKAGE_JSON) { - let (source, path): (&[u8], &ZStr) = - if matches!(manager.options.patch_features, PatchFeatures::Commit { .. }) { - 'source_and_path: { - let root_package_json_entry = match manager - .workspace_package_json_cache - .get_with_path( - manager.log_mut(), - root_package_json_path.as_bytes(), - GetJSONOptions::default(), - ) - .unwrap() - { - Ok(e) => e, - Err(err) => { - Output::err( - err, - "failed to read/parse package.json at '{s}'", - (BStr::new(root_package_json_path.as_bytes()),), - ); - Global::exit(1); - } - }; - - break 'source_and_path ( - &root_package_json_entry.source.contents, - root_package_json_path, - ); - } - } else { - ( - &new_package_json_source, - manager.original_package_json_path.as_zstr(), - ) - }; + // `bun patch --commit` records the patch in the root package.json, even when run from a workspace. + let path: &ZStr = if matches!(manager.options.patch_features, PatchFeatures::Commit { .. }) + { + root_package_json_path + } else { + manager.original_package_json_path.as_zstr() + }; + // Written from the cache entry: a pnpm migration inside `install_with_manager` may have edited it since. + let entry = match manager + .workspace_package_json_cache + .get_with_path( + manager.log_mut(), + path.as_bytes(), + GetJSONOptions::default(), + ) + .unwrap() + { + Ok(entry) => entry, + Err(err) => { + Output::err( + err, + "failed to read/parse package.json at '{s}'", + (BStr::new(path.as_bytes()),), + ); + Global::exit(1); + } + }; + let source: &[u8] = &entry.source.contents; // Now that we've run the install step // We can save our in-memory package.json to disk diff --git a/src/install/PackageManager/workspace_manifests.rs b/src/install/PackageManager/workspace_manifests.rs index aeade57954aa..71a03100e741 100644 --- a/src/install/PackageManager/workspace_manifests.rs +++ b/src/install/PackageManager/workspace_manifests.rs @@ -1,9 +1,12 @@ +use core::fmt; + use bstr::BStr; use bun_collections::HashMap; use bun_core::{Global, Output}; use crate::dependency::{Behavior, Tag as DependencyTag}; -use crate::lockfile::{Lockfile, Package}; +use crate::lockfile::package::folder_relative_to_top_level_dir; +use crate::lockfile::{DependencySlice, Lockfile, Package}; use crate::{Features, PackageNameHash}; use super::PackageManager; @@ -11,6 +14,7 @@ use super::add_remove_with_filter::{WorkspaceTarget, fetch_entry, root_package_j use super::workspace_selection::WorkspaceGraph; /// Root + member package.json files parsed the way `bun install` parses them, into a throw-away lockfile. +/// Errors the parse only logs (an invalid catalog range, say) fail it here, as they fail `bun install`. pub(crate) struct ScratchManifests { pub(crate) lockfile: Lockfile, pub(crate) log: bun_ast::Log, @@ -27,7 +31,12 @@ impl ScratchManifests { } /// Must run first: it fills `lockfile.workspace_paths`, which `workspace:` rows in every file resolve through. - pub(crate) fn parse_root(&mut self, manager: &mut PackageManager) -> crate::Result<()> { + /// `features` must include `workspaces` for that, and `is_main` for the catalogs. + pub(crate) fn parse_root( + &mut self, + manager: &mut PackageManager, + features: Features, + ) -> crate::Result<()> { let root_target = WorkspaceTarget { name: Box::default(), name_hash: None, @@ -46,8 +55,9 @@ impl ScratchManifests { &root_source, root_json, &mut resolver, - Features::main(), - ) + features, + )?; + self.fail_on_logged_errors() } pub(crate) fn parse_member( @@ -71,8 +81,99 @@ impl ScratchManifests { &mut resolver, Features::WORKSPACE, )?; + self.fail_on_logged_errors()?; Ok(pkg) } + + fn fail_on_logged_errors(&self) -> crate::Result<()> { + if self.log.has_errors() { + return Err(crate::Error::InstallFailed); + } + Ok(()) + } +} + +/// The workspace versions and catalogs `bun pm pack` / `bun publish` substitute, read from the package.json +/// files as they are now (bun.lock has the versions of the last install; releases bump them after it). +pub struct WorkspaceManifests { + lockfile: Lockfile, + /// Has a `Behavior::WORKSPACE` entry per workspace: its name and root-relative directory. + root_dependencies: DependencySlice, + root_package_json_path: Box<[u8]>, +} + +impl WorkspaceManifests { + /// Exits with `bun install`'s errors when the root package.json or a workspace does not parse. + pub fn load(manager: &mut PackageManager) -> WorkspaceManifests { + // Only what pack reads, so `bun install`'s checks on the root's own dependency lists do not fail a pack. + let features = Features { + is_main: true, + workspaces: true, + dependencies: false, + peer_dependencies: false, + ..Features::default() + }; + let mut scratch = ScratchManifests::new(); + if let Err(err) = scratch.parse_root(manager, features) { + crash( + &mut scratch.log, + err, + format_args!("failed to read the workspace's package.json files"), + ); + } + WorkspaceManifests { + lockfile: scratch.lockfile, + root_dependencies: scratch.root.dependencies, + root_package_json_path: root_package_json_path(), + } + } + + /// Whether `name` is one of the workspaces (the root package is not one). + pub fn has_workspace(&self, name: &[u8]) -> bool { + let name_hash: PackageNameHash = bun_semver::string::Builder::string_hash(name); + self.lockfile.workspace_paths.contains(&name_hash) + } + + /// The workspace `workspace:` in `package_dir` links, per `Package::parse_dependency`. + pub fn workspace_name_at_path(&self, package_dir: &[u8], path: &[u8]) -> Option<&[u8]> { + // Joined as a path, `workspace:` alone would name `package_dir` itself. + if path.is_empty() { + return None; + } + let mut buf = bun_paths::path_buffer_pool::get(); + let relative_directory = folder_relative_to_top_level_dir(package_dir, path, &mut buf[..])?; + let string_buf = self.lockfile.buffers.string_bytes.as_slice(); + self.root_dependencies + .get(self.lockfile.buffers.dependencies.as_slice()) + .iter() + .find(|dependency| { + dependency.behavior.is_workspace() + && dependency.version.workspace().slice(string_buf) == relative_directory + }) + .map(|dependency| dependency.name.slice(string_buf)) + } + + /// The package.json whose `workspaces` and catalogs these are (the packed package's own when it is no workspace). + pub fn root_package_json_path(&self) -> &[u8] { + &self.root_package_json_path + } + + /// The (semver) `version` in the package.json of the workspace named `name`, if any. + pub fn workspace_version(&self, name: &[u8]) -> Option { + let name_hash: PackageNameHash = bun_semver::string::Builder::string_hash(name); + let version = self.lockfile.workspace_versions.get(&name_hash)?; + Some(version.fmt(self.lockfile.buffers.string_bytes.as_slice())) + } + + /// The range catalog `catalog_name` (`""` or `"default"` for the default one) declares for `dependency_name`, as written. + pub fn catalog_version(&self, catalog_name: &[u8], dependency_name: &[u8]) -> Option<&[u8]> { + let string_buf = self.lockfile.buffers.string_bytes.as_slice(); + let dependency = self + .lockfile + .catalogs + .find(string_buf, catalog_name, dependency_name)?; + Some(dependency.version.literal.slice(string_buf)) + } } /// Graph index i == `targets[i]`; the target whose `name_hash` is `None` is the root. @@ -82,8 +183,8 @@ pub(crate) fn relation_graph( pattern: &[u8], ) -> WorkspaceGraph { let mut scratch = ScratchManifests::new(); - if let Err(err) = scratch.parse_root(manager) { - crash(&mut scratch.log, pattern, err); + if let Err(err) = scratch.parse_root(manager, Features::main()) { + crash_for_filter(&mut scratch.log, pattern, err); } let mut parsed: Vec<(u32, Package)> = Vec::with_capacity(targets.len()); @@ -94,7 +195,7 @@ pub(crate) fn relation_graph( } match scratch.parse_member(manager, target) { Ok(pkg) => parsed.push((i as u32, pkg)), - Err(err) => crash(&mut scratch.log, pattern, err), + Err(err) => crash_for_filter(&mut scratch.log, pattern, err), } } @@ -150,14 +251,23 @@ pub(crate) fn relation_graph( WorkspaceGraph::from_edges(targets.len(), edges) } -fn crash(log: &mut bun_ast::Log, pattern: &[u8], err: crate::Error) -> ! { +fn crash_for_filter(log: &mut bun_ast::Log, pattern: &[u8], err: crate::Error) -> ! { + crash( + log, + err, + format_args!( + "failed to read the workspace dependencies for --filter \"{}\"", + BStr::new(pattern) + ), + ) +} + +/// The parse errors explain the failure when there are any; `what` and `err` are the fallback. +fn crash(log: &mut bun_ast::Log, err: crate::Error, what: fmt::Arguments<'_>) -> ! { if log.has_errors() { let _ = log.print(std::ptr::from_mut(Output::error_writer())); } else { - Output::err_generic( - "failed to read the workspace dependencies for --filter \"{}\": {}", - (BStr::new(pattern), err.name()), - ); + Output::err_generic("{}: {}", (what, err.name())); } Global::crash(); } diff --git a/src/install/PackageManifestMap.rs b/src/install/PackageManifestMap.rs index cdd492b83b82..4c52cc1e3ac2 100644 --- a/src/install/PackageManifestMap.rs +++ b/src/install/PackageManifestMap.rs @@ -74,11 +74,22 @@ impl PackageManifestMap { ) } + /// When a package was requested both abbreviated and extended in one install, the extended + /// response wins whichever arrives last, or dependencies waiting on it would never resolve. pub(crate) fn insert( &mut self, name_hash: PackageNameHash, manifest: npm::PackageManifest, ) -> Result<(), bun_alloc::AllocError> { + if !manifest.pkg.has_extended_manifest + && matches!( + self.hash_map.get(&name_hash), + Some(Value::Manifest(existing)) + if existing.pkg.has_extended_manifest && existing.name() == manifest.name() + ) + { + return Ok(()); + } self.hash_map.insert(name_hash, Value::Manifest(manifest)); Ok(()) } @@ -144,6 +155,8 @@ impl PackageManifestMap { /// `timestamp_for_manifest_cache_control`) are hoisted into /// [`DiskCacheCtx`] so callers never hold `&mut pm.manifests` and a /// `PackageManager` borrow simultaneously. + /// + /// With `needs_extended_manifest`, an abbreviated entry reads as missing but stays for other callers. pub(crate) fn by_name_hash_allow_expired( &mut self, ctx: DiskCacheCtx, @@ -154,11 +167,15 @@ impl PackageManifestMap { cache_behavior: CacheBehavior, needs_extended_manifest: bool, ) -> Option<&mut npm::PackageManifest> { + let usable = |m: &npm::PackageManifest| { + m.name() == name && (!needs_extended_manifest || m.pkg.has_extended_manifest) + }; + if cache_behavior == CacheBehavior::LoadFromMemory { let entry = self.hash_map.get_mut(&name_hash)?; return match entry { - Value::Manifest(m) if m.name() == name => Some(m), - Value::Expired(m) if m.name() == name => { + Value::Manifest(m) if usable(m) => Some(m), + Value::Expired(m) if usable(m) => { if let Some(expiry) = is_expired { *expiry = true; Some(m) @@ -170,90 +187,42 @@ impl PackageManifestMap { }; } - match self.hash_map.entry(name_hash) { - Entry::Occupied(occ) => { - let value_ptr = occ.into_mut(); - if let Value::Manifest(m) | Value::Expired(m) = &*value_ptr { - if m.name() != name { - return None; - } - } - // Compute the demote decision first without holding a borrow - // that escapes the fn. - let demote = matches!( - value_ptr, - Value::Manifest(m) - if needs_extended_manifest && !m.pkg.has_extended_manifest - ); - if demote { - let Value::Manifest(m) = core::mem::replace(value_ptr, Value::NotFound) else { - unreachable!() - }; - *value_ptr = Value::Expired(m); - } else if let Value::Manifest(m) = value_ptr { - return Some(m); - } - - if let Some(expiry) = is_expired { - if let Value::Expired(m) = value_ptr { - *expiry = true; - return Some(m); - } - } - - None - } + let value_ptr = match self.hash_map.entry(name_hash) { + Entry::Occupied(occ) => occ.into_mut(), Entry::Vacant(vac) => { + let mut loaded = None; if ctx.enable_manifest_cache { // `ctx.cache_directory` is `Some` iff `enable_manifest_cache` // (see `manifest_disk_cache_ctx`). let cache_fd = ctx.cache_directory.expect("cache_directory"); - if let Some(manifest) = npm::package_manifest::Serializer::load_by_file_id( + loaded = npm::package_manifest::Serializer::load_by_file_id( scope, cache_fd, name, name_hash, ) .ok() - .flatten() - { - if needs_extended_manifest && !manifest.pkg.has_extended_manifest { - let value_ptr = vac.insert(Value::Expired(manifest)); - if let Some(expiry) = is_expired { - *expiry = true; - let Value::Expired(m) = value_ptr else { - unreachable!() - }; - return Some(m); - } - return None; - } - + .flatten(); + } + vac.insert(match loaded { + Some(manifest) if ctx.enable_manifest_cache_control && manifest.pkg.public_max_age - > ctx.timestamp_for_manifest_cache_control - { - let value_ptr = vac.insert(Value::Manifest(manifest)); - let Value::Manifest(m) = value_ptr else { - unreachable!() - }; - return Some(m); - } else { - let value_ptr = vac.insert(Value::Expired(manifest)); - - if let Some(expiry) = is_expired { - *expiry = true; - let Value::Expired(m) = value_ptr else { - unreachable!() - }; - return Some(m); - } - - return None; - } + > ctx.timestamp_for_manifest_cache_control => + { + Value::Manifest(manifest) } - } - - vac.insert(Value::NotFound); - None + Some(manifest) => Value::Expired(manifest), + None => Value::NotFound, + }) } + }; + + match value_ptr { + Value::Manifest(m) if usable(m) => Some(m), + Value::Expired(m) if usable(m) => { + let expiry = is_expired?; + *expiry = true; + Some(m) + } + _ => None, } } } diff --git a/src/install/TarballStream.rs b/src/install/TarballStream.rs index 7ae9f698469b..aa1caa042fd6 100644 --- a/src/install/TarballStream.rs +++ b/src/install/TarballStream.rs @@ -133,6 +133,8 @@ pub struct TarballStream { /// Owned copy of the temp-directory name. // `ZBox` is the owned NUL-terminated counterpart of `&ZStr`. tmpname: ZBox, + /// Captured in `init`, before the first byte is extracted. + cache_publish: CachePublish, /// Incremental SHA over the *compressed* bytes, matching /// `Integrity.verify` / `Integrity.forBytes` in the buffered path. @@ -230,6 +232,7 @@ impl TarballStream { }, compute_if_missing, ); + let cache_publish = tarball.cache_publish(); // bun.TrivialNew(@This()) → heap::alloc(Box::new(...)). Pointer is // recovered via `container_of` from the thread-pool callback and @@ -254,6 +257,7 @@ impl TarballStream { entry_final_offset: 0, dest: None, tmpname: ZBox::from_bytes(b""), + cache_publish, hasher, resolved_github_dirname: b"", want_first_dirname, @@ -1089,7 +1093,7 @@ impl TarballStream { bun_ast::Loc::EMPTY, format_args!( "Refusing to install package with invalid name \"{}\"", - bun_fmt::s(tarball.name_and_basename().0), + bun_fmt::escape_control_chars(tarball.name_and_basename().0), ), ); } else { @@ -1153,12 +1157,15 @@ impl TarballStream { let (name, basename) = tarball.name_and_basename(); + let integrity = tarball.lockfile_integrity(|| self.hasher.final_()); let mut result = match tarball.move_to_cache_directory( &mut (*task).log, self.tmpname.as_zstr(), name, basename, self.resolved_github_dirname, + &integrity, + self.cache_publish, ) { Ok(r) => r, Err(err) => { @@ -1167,19 +1174,7 @@ impl TarballStream { return; } }; - - match tarball.resolution.tag { - ResolutionTag::Github - | ResolutionTag::RemoteTarball - | ResolutionTag::LocalTarball => { - if tarball.integrity.tag.is_supported() { - result.integrity = tarball.integrity; - } else { - result.integrity = self.hasher.final_(); - } - } - _ => {} - } + result.integrity = integrity; if PackageManager::verbose_install() { bun_core::pretty_errorln!( @@ -1452,5 +1447,6 @@ fn tokenize_rest_after_first(s: &[OSPathChar]) -> &[OSPathChar] { // Resolved Phase-B paths: Resolution::Tag is the real npm/git/tarball // discriminant; Data/Status live on PackageManagerTask. +use crate::extract_tarball::CachePublish; use crate::package_manager_task::{Data as TaskData, Status as TaskStatus}; use crate::resolution::Tag as ResolutionTag; diff --git a/src/install/audit_fix.rs b/src/install/audit_fix.rs index 0fd57cc46dec..200d7653d118 100644 --- a/src/install/audit_fix.rs +++ b/src/install/audit_fix.rs @@ -3,13 +3,16 @@ use core::mem::ManuallyDrop; use std::io::Write as _; use bstr::BStr; +use bun_ast::Expr; use bun_collections::{DynamicBitSet, HashMap, index_sort}; +use bun_core::fmt::escape_control_chars; use bun_core::{Global, Output, UnwrapOrOom as _, pretty, prettyln, strings}; use bun_semver::query::Group; use bun_semver::{self as Semver, SlicedString}; use crate::dependency::Behavior; use crate::lockfile::Lockfile; +use crate::lockfile::override_map::OverrideRule; use crate::lockfile::package::PackageColumns as _; use crate::npm::PackageManifest; use crate::package_manager::Options::{Do, Enable, LogLevel}; @@ -23,7 +26,7 @@ use crate::{ mod json; mod package_json_edits; -pub use package_json_edits::PackageJsonEdit; +pub use package_json_edits::{EditSite, OverrideSelector, PackageJsonEdit}; pub struct Advisory { pub package_name: Box<[u8]>, @@ -52,11 +55,15 @@ pub struct PlannedEdge { pub parent: PackageID, } +#[derive(PartialEq, Eq)] pub struct Blocker { + /// The dependent package or importer file; `package.json` when `override_rule` is set, since the range then comes from its `overrides`. pub dependent: Box<[u8]>, pub range: Box<[u8]>, pub bundled: bool, pub latest_fixes: bool, + /// Key of the `overrides` rule that holds the version, in the form `OverrideSelector::key` produces. + pub override_rule: Option>, } pub struct BlockedFix { @@ -183,9 +190,12 @@ pub struct FixOutcome { struct Edge { dep_id: DependencyID, parent: PackageID, + /// What the edge resolves against once overrides and catalogs are applied; `None` unless that is an npm range. range: Option, + /// The override rule's value when one applies, else the dependent's own specifier; shown when `range` is `None`. literal: Box<[u8]>, dependent: Box<[u8]>, + override_rule: Option>, bundled: bool, peer: bool, latest_fixes: bool, @@ -244,7 +254,8 @@ fn print_elapsed_line() { fn print_tokens(tokens: &[Box<[u8]>]) { for (i, token) in tokens.iter().enumerate() { - pretty!("{}{}", if i > 0 { ", " } else { "" }, BStr::new(token)); + let token = escape_control_chars(token); + pretty!("{}{}", if i > 0 { ", " } else { "" }, token); } } @@ -299,14 +310,14 @@ pub fn print_unaudited(groups: &[UnauditedRegistry]) { bun_core::warn!( "{} did not answer the audit request; skipped {}", bun_core::fmt::redacted_npm_url(&group.registry), - BStr::new(&packages) + escape_control_chars(&packages) ); } else { bun_core::warn!( "{} did not answer the audit request ({}); skipped {}", bun_core::fmt::redacted_npm_url(&group.registry), BStr::new(&group.reason), - BStr::new(&packages) + escape_control_chars(&packages) ); } } @@ -428,41 +439,38 @@ fn print_manifest_unavailable(manager: &PackageManager, items: &[ManifestUnavail Output::flush(); } +/// The root/workspace package.json (or catalog, or override rule) entry whose rewrite would move this edge: an exact pin, or with `latest` any npm range. fn pin_for( lockfile: &Lockfile, - dep_id: usize, + root_json: Option<&Expr>, dep: &Dependency, parent: PackageID, + rule: Option>, latest: bool, ) -> Option { - if parent == invalid_package_id || dep.behavior.is_bundled() || dep.behavior.is_workspace() { + if dep.behavior.is_bundled() || dep.behavior.is_workspace() { return None; } let buf = lockfile.buffers.string_bytes.as_slice(); - let res = lockfile.packages.items_resolution(); - let parent_res = &res[parent as usize]; - if !matches!( - parent_res.tag, - ResolutionTag::Root | ResolutionTag::Workspace - ) { - return None; - } - let is_alias = dep.version.tag == DependencyVersionTag::Npm && dep.version.npm().is_alias; - if !is_alias - && lockfile - .overrides - .get(lockfile, dep_id as DependencyID, dep.name_hash) - .is_some() - { - return None; - } - let key: Box<[u8]> = Box::from(dep.name.slice(buf)); - match dep.version.tag { + let name = dep.name.slice(buf); + let version = match rule { + Some(rule) => rule.version(), + None => { + if parent == invalid_package_id + || !matches!( + lockfile.packages.items_resolution()[parent as usize].tag, + ResolutionTag::Root | ResolutionTag::Workspace + ) + { + return None; + } + &dep.version + } + }; + match version.tag { DependencyVersionTag::Catalog => { - let catalog_name = dep.version.catalog().slice(buf); - let entry = lockfile - .catalogs - .find(buf, catalog_name, dep.name.slice(buf))?; + let catalog_name = version.catalog().slice(buf); + let entry = lockfile.catalogs.find(buf, catalog_name, name)?; if entry.version.tag != DependencyVersionTag::Npm || (!latest && entry.version.npm().version.get_exact_version().is_none()) { @@ -471,24 +479,44 @@ fn pin_for( Some(PackageJsonEdit { owner: 0, file: Box::from(&b"package.json"[..]), - catalog: Some(Box::from(catalog_name)), - key, + site: EditSite::Catalog(Box::from(catalog_name)), + key: Box::from(name), old_literal: Box::from(entry.version.literal.slice(buf)), new_literal: Box::default(), }) } DependencyVersionTag::Npm => { if !latest { - dep.version.npm().version.get_exact_version()?; + version.npm().version.get_exact_version()?; + } + let literal = version.literal.slice(buf); + match rule { + Some(rule) => { + let selector = OverrideSelector::from_rule(rule, buf); + let written = + package_json_edits::override_literal(root_json?, name, &selector)?; + // Differs when package.json spells the value as a `$ref` to a dependency; bun.lock stores what it resolved to. + if *written != *strings::trim(literal, &strings::WHITESPACE_CHARS) { + return None; + } + Some(PackageJsonEdit { + owner: 0, + file: Box::from(&b"package.json"[..]), + site: EditSite::Override(selector), + key: Box::from(name), + old_literal: written, + new_literal: Box::default(), + }) + } + None => Some(PackageJsonEdit { + owner: parent, + file: importer_file(lockfile, parent)?, + site: EditSite::Dependencies, + key: Box::from(name), + old_literal: Box::from(literal), + new_literal: Box::default(), + }), } - Some(PackageJsonEdit { - owner: parent, - file: importer_file(lockfile, parent)?, - catalog: None, - key, - old_literal: Box::from(dep.version.literal.slice(buf)), - new_literal: Box::default(), - }) } _ => None, } @@ -502,15 +530,12 @@ pub fn plan_fixes(manager: &mut PackageManager, advisories: &[Advisory]) -> crat let mut instances: Vec = Vec::new(); let mut checked_names: HashMap = HashMap::new(); - { + let instance_of: Vec = { let lockfile = &*manager.lockfile; let buf = lockfile.buffers.string_bytes.as_slice(); let names = lockfile.packages.items_name(); let name_hashes = lockfile.packages.items_name_hash(); let res = lockfile.packages.items_resolution(); - let dep_slices = lockfile.packages.items_dependencies(); - let deps = lockfile.buffers.dependencies.as_slice(); - let resolutions = lockfile.buffers.resolutions.as_slice(); let mut instance_of: Vec = vec![u32::MAX; res.len()]; for pkg_id in 0..res.len() { @@ -544,49 +569,63 @@ pub fn plan_fixes(manager: &mut PackageManager, advisories: &[Advisory]) -> crat edges: Vec::new(), }); } + instance_of + }; - if !instances.is_empty() { - let mut parent_of: Vec = vec![invalid_package_id; deps.len()]; - for (pkg_id, slice) in dep_slices.iter().enumerate() { - let end = (slice.end() as usize).min(deps.len()); - for slot in &mut parent_of[(slice.begin() as usize).min(end)..end] { - *slot = pkg_id as PackageID; - } + if !instances.is_empty() { + let root_json: Option = (!manager.lockfile.overrides.is_empty()) + .then(|| package_json_edits::root_package_json(manager)); + let lockfile = &*manager.lockfile; + let buf = lockfile.buffers.string_bytes.as_slice(); + let deps = lockfile.buffers.dependencies.as_slice(); + let mut parent_of: Vec = vec![invalid_package_id; deps.len()]; + for (pkg_id, slice) in lockfile.packages.items_dependencies().iter().enumerate() { + let end = (slice.end() as usize).min(deps.len()); + for slot in &mut parent_of[(slice.begin() as usize).min(end)..end] { + *slot = pkg_id as PackageID; } + } - for (dep_id, &target) in resolutions.iter().enumerate() { - if target == invalid_package_id { - continue; - } - let Some(&instance) = instance_of.get(target as usize) else { - continue; - }; - if instance == u32::MAX || deps[dep_id].behavior.is_optional_peer() { - continue; - } - let dep = &deps[dep_id]; - let parent = parent_of[dep_id]; - let mut pin = pin_for(lockfile, dep_id, dep, parent, true); - let latest_fixes = !latest && pin.is_some(); - if latest_fixes { - pin = pin_for(lockfile, dep_id, dep, parent, false); - } - instances[instance as usize].edges.push(Edge { - dep_id: dep_id as DependencyID, - parent, - range: crate::dedupe::effective_npm_range( - lockfile, - dep_id as DependencyID, - dep, - ), - literal: Box::from(dep.version.literal.slice(buf)), - dependent: dependent_label(lockfile, parent), - bundled: dep.behavior.is_bundled(), - peer: dep.behavior.is_peer(), - latest_fixes, - pin, - }); + for (dep_id, &target) in lockfile.buffers.resolutions.iter().enumerate() { + if target == invalid_package_id { + continue; + } + let Some(&instance) = instance_of.get(target as usize) else { + continue; + }; + if instance == u32::MAX || deps[dep_id].behavior.is_optional_peer() { + continue; + } + let dep = &deps[dep_id]; + let parent = parent_of[dep_id]; + // A bundled copy ships inside its dependent whatever an override says, so it stays attributed to the dependent. + let rule = if dep.behavior.is_bundled() { + None + } else { + crate::dedupe::applied_override(lockfile, dep_id as DependencyID, dep) + }; + let mut pin = pin_for(lockfile, root_json.as_ref(), dep, parent, rule, true); + let latest_fixes = !latest && pin.is_some(); + if latest_fixes { + pin = pin_for(lockfile, root_json.as_ref(), dep, parent, rule, false); } + instances[instance as usize].edges.push(Edge { + dep_id: dep_id as DependencyID, + parent, + range: crate::dedupe::effective_npm_range(lockfile, dep_id as DependencyID, dep), + literal: Box::from( + rule.map_or(&dep.version, OverrideRule::version) + .literal + .slice(buf), + ), + dependent: dependent_label(lockfile, parent), + override_rule: rule + .map(|rule| OverrideSelector::from_rule(rule, buf).key(dep.name.slice(buf))), + bundled: dep.behavior.is_bundled(), + peer: dep.behavior.is_peer(), + latest_fixes, + pin, + }); } } @@ -691,7 +730,16 @@ pub fn plan_fixes(manager: &mut PackageManager, advisories: &[Advisory]) -> crat } let upgrade_count = candidates.len(); for (i, &v) in releases.iter().enumerate().rev() { - if v.order(inst.current, manifest_buf, buf) == Ordering::Less && is_safe(v) { + if v.order(inst.current, manifest_buf, buf) != Ordering::Less { + continue; + } + // Downgrades stay in the installed major (minor below 1.0.0): an older line is a different API, + // and the bulk response only has advisories for the installed versions' lines (#39309). + let cur = inst.current; + if v.major != cur.major || (v.major == 0 && v.minor != cur.minor) { + break; + } + if is_safe(v) { candidates.push(Candidate { version: v, index: i, @@ -819,7 +867,8 @@ pub fn plan_fixes(manager: &mut PackageManager, advisories: &[Advisory]) -> crat edits.push(edit); } // A rewritten peer row is deferred by the differ and rebinds to the old package unless the edge is pinned too. - if edge.peer { + // A rewritten range re-resolves to the newest release it allows, which for a downgrade is the vulnerable one again (`^1.0.1` still takes 1.1.0). + if edge.peer || candidate.downgrade { edges.push(PlannedEdge { dep_id: edge.dep_id, parent: edge.parent, @@ -855,13 +904,14 @@ pub fn plan_fixes(manager: &mut PackageManager, advisories: &[Advisory]) -> crat }); } - let blockers: Vec = inst - .edges - .iter() - .zip(&target) - .filter(|(_, t)| t.is_none()) - .map(|(edge, _)| Blocker { - dependent: edge.dependent.clone(), + let mut blockers: Vec = Vec::new(); + for (edge, _) in inst.edges.iter().zip(&target).filter(|(_, t)| t.is_none()) { + let blocker = Blocker { + dependent: if edge.override_rule.is_some() { + Box::from(&b"package.json"[..]) + } else { + edge.dependent.clone() + }, range: if edge.bundled { inst.from.clone() } else { @@ -872,8 +922,13 @@ pub fn plan_fixes(manager: &mut PackageManager, advisories: &[Advisory]) -> crat }, bundled: edge.bundled, latest_fixes: edge.latest_fixes, - }) - .collect(); + override_rule: edge.override_rule.clone(), + }; + // Every edge a rule holds reports the same blocker. + if !blockers.contains(&blocker) { + blockers.push(blocker); + } + } if blockers.is_empty() { expected_gone.push((inst.name_hash, inst.from)); continue; @@ -942,10 +997,18 @@ impl FixPlan { prettyln!(""); for edit in &fix.edits { pretty!(" {}", BStr::new(&edit.file)); - match edit.catalog.as_deref() { - None => {} - Some(b"" | b"default") => pretty!(" (catalog)"), - Some(catalog) => pretty!(" (catalog {})", BStr::new(catalog)), + match &edit.site { + EditSite::Dependencies => {} + EditSite::Catalog(catalog) => match &**catalog { + b"" | b"default" => pretty!(" (catalog)"), + catalog => pretty!(" (catalog {})", BStr::new(catalog)), + }, + EditSite::Override(selector) if selector.is_bare() => { + pretty!(" (overrides)"); + } + EditSite::Override(selector) => { + pretty!(" (overrides {})", BStr::new(&selector.key(&edit.key))); + } } prettyln!( ": {} {} {}", @@ -977,17 +1040,24 @@ impl FixPlan { } prettyln!(""); for blocker in &item.blockers { - prettyln!( + pretty!( " {} {} {}@{}", BStr::new(&blocker.dependent), if blocker.bundled { "bundles" + } else if blocker.override_rule.is_some() { + "overrides" } else { "depends on" }, BStr::new(&item.name), BStr::new(&blocker.range) ); + match &blocker.override_rule { + Some(rule) if **rule != *item.name => pretty!(" ({})", BStr::new(rule)), + _ => {} + } + prettyln!(""); } if item.blockers.iter().any(|blocker| blocker.latest_fixes) { prettyln!(" bun audit fix --latest"); @@ -1010,7 +1080,7 @@ impl FixPlan { } pretty!(" bun audit fix"); for token in &all_tokens { - pretty!(" --ignore {}", BStr::new(token)); + pretty!(" --ignore {}", escape_control_chars(token)); } prettyln!(""); prettyln!(""); @@ -1018,7 +1088,8 @@ impl FixPlan { if !self.unmatched.is_empty() { prettyln!("not matched to an installed version:"); for item in &self.unmatched { - prettyln!(" {}@{}", BStr::new(&item.name), BStr::new(&item.range)); + let name = escape_control_chars(&item.name); + prettyln!(" {}@{}", name, escape_control_chars(&item.range)); } prettyln!(""); } @@ -1199,18 +1270,15 @@ pub fn prepare_install(manager: &mut PackageManager, plan: &FixPlan) -> crate::R package_json_edits::apply(manager, plan)?; if plan.fixes.iter().any(|fix| fix.too_recent) { - let mut names: Vec<&'static [u8]> = manager - .options - .minimum_release_age_excludes - .map_or_else(Vec::new, <[_]>::to_vec); - names.extend( + let excludes = crate::npm::MinimumReleaseAgeExcludes::with_packages( + manager.options.minimum_release_age_excludes, plan.fixes .iter() .filter(|fix| fix.too_recent) .map(|fix| &*bun_core::heap::release(fix.name.clone())), ); manager.options.minimum_release_age_excludes = - Some(&*bun_core::heap::release(names.into_boxed_slice())); + Some(&*bun_core::heap::release(Box::new(excludes))); } manager.audit_fix_pins = plan diff --git a/src/install/audit_fix/json.rs b/src/install/audit_fix/json.rs index 7e0a8dae8165..b6c26fc50fce 100644 --- a/src/install/audit_fix/json.rs +++ b/src/install/audit_fix/json.rs @@ -2,7 +2,7 @@ use std::io::Write as _; use bun_core::Output; -use super::{FixOutcome, FixPlan, PackageJsonEdit}; +use super::{EditSite, FixOutcome, FixPlan, PackageJsonEdit}; pub(super) fn write(plan: &FixPlan, outcome: Option<&FixOutcome>, dry_run: bool) { let (fixed, remaining) = match outcome { @@ -59,7 +59,12 @@ pub(super) fn write(plan: &FixPlan, outcome: Option<&FixOutcome>, dry_run: bool) s(&mut out, &blocker.dependent); out.extend_from_slice(b",\"range\":"); s(&mut out, &blocker.range); - let _ = write!(out, ",\"bundled\":{}}}", blocker.bundled); + let _ = write!(out, ",\"bundled\":{},\"override\":", blocker.bundled); + match &blocker.override_rule { + None => out.extend_from_slice(b"null"), + Some(rule) => s(&mut out, rule), + } + out.push(b'}'); } out.extend_from_slice(b"]}"); } @@ -146,10 +151,15 @@ fn write_edit(out: &mut Vec, edit: &PackageJsonEdit) { out.extend_from_slice(b"{\"file\":"); s(out, &edit.file); out.extend_from_slice(b",\"catalog\":"); - match edit.catalog.as_deref() { - None => out.extend_from_slice(b"null"), - Some(b"") => out.extend_from_slice(b"\"default\""), - Some(catalog) => s(out, catalog), + match &edit.site { + EditSite::Catalog(catalog) if catalog.is_empty() => out.extend_from_slice(b"\"default\""), + EditSite::Catalog(catalog) => s(out, catalog), + EditSite::Dependencies | EditSite::Override(_) => out.extend_from_slice(b"null"), + } + out.extend_from_slice(b",\"override\":"); + match &edit.site { + EditSite::Override(selector) => s(out, &selector.key(&edit.key)), + EditSite::Dependencies | EditSite::Catalog(_) => out.extend_from_slice(b"null"), } out.extend_from_slice(b",\"key\":"); s(out, &edit.key); diff --git a/src/install/audit_fix/package_json_edits.rs b/src/install/audit_fix/package_json_edits.rs index 294acbfd0d29..494569ade597 100644 --- a/src/install/audit_fix/package_json_edits.rs +++ b/src/install/audit_fix/package_json_edits.rs @@ -8,7 +8,11 @@ use bun_semver::{PinnedVersion, Version}; use crate::bun_fs::FileSystem; use crate::lockfile::CatalogMap; +use crate::lockfile::override_map::OverrideRule; use crate::lockfile::package::PackageColumns as _; +use crate::lockfile_real::override_selector::{ + PackageSelector, Selector, parse_package_segment, parse_selector, +}; use crate::package_manager_real::add_remove_with_filter::{ WorkspaceTarget, fetch_entry_root, root_package_json_path, store_entry, }; @@ -27,16 +31,91 @@ const DEPENDENCY_GROUPS: [&[u8]; 4] = [ pub struct PackageJsonEdit { pub owner: PackageID, pub file: Box<[u8]>, - pub catalog: Option>, + pub site: EditSite, + /// The package whose range is rewritten. pub key: Box<[u8]>, pub old_literal: Box<[u8]>, pub new_literal: Box<[u8]>, } +#[derive(Clone, PartialEq, Eq)] +pub enum EditSite { + /// `key` in whichever of the four dependency groups of `file` declares `old_literal`. + Dependencies, + /// `key` in this catalog of the root package.json; empty means the default catalog. + Catalog(Box<[u8]>), + /// The rule for `key` in the root package.json's `overrides` (or `resolutions`). + Override(OverrideSelector), +} + +/// A rule of the root package.json's `overrides`/`resolutions` as `OverrideMap` normalizes it, minus the target name (the edit's `key`). +#[derive(Clone, PartialEq, Eq)] +pub struct OverrideSelector { + /// Name and declared range of the dependent the rule is scoped to; the range is empty when any version of it qualifies. + pub parent: Option<(Box<[u8]>, Box<[u8]>)>, + /// Empty when the rule applies whatever range the dependent declares. + pub target_range: Box<[u8]>, +} + +impl OverrideSelector { + pub(super) fn from_rule(rule: OverrideRule<'_>, buf: &[u8]) -> OverrideSelector { + let OverrideRule::Scoped(rule) = rule else { + return OverrideSelector { + parent: None, + target_range: Box::default(), + }; + }; + let text = |s: &bun_semver::String| Box::from(s.slice(buf)); + let parent = rule.parent.as_ref(); + OverrideSelector { + parent: parent.map(|p| (text(&p.name), text(&p.version.literal))), + target_range: text(&rule.target_range.literal), + } + } + + /// A plain `"name": ...` rule, as opposed to one scoped to a parent or to a declared range. + pub fn is_bare(&self) -> bool { + self.parent.is_none() && self.target_range.is_empty() + } + + /// The rule's key in the `parent@range>name@range` form bun.lock writes, which `overrides` also accepts. + pub fn key(&self, name: &[u8]) -> Box<[u8]> { + let mut out: Vec = Vec::new(); + if let Some((parent, parent_range)) = &self.parent { + push_segment(&mut out, parent, parent_range); + out.push(b'>'); + } + push_segment(&mut out, name, &self.target_range); + out.into_boxed_slice() + } + + fn matches( + &self, + name: &[u8], + parent: Option, + target: PackageSelector, + ) -> bool { + let own_parent = self + .parent + .as_ref() + .map(|(name, range)| (&**name, &**range)); + (target.name, target.range) == (name, &*self.target_range) + && parent.map(|p| (p.name, p.range)) == own_parent + } +} + +fn push_segment(out: &mut Vec, name: &[u8], range: &[u8]) { + out.extend_from_slice(name); + if !range.is_empty() { + out.push(b'@'); + out.extend_from_slice(range); + } +} + impl PackageJsonEdit { pub(crate) fn same_site(&self, other: &PackageJsonEdit) -> bool { self.owner == other.owner - && self.catalog == other.catalog + && self.site == other.site && self.key == other.key && self.old_literal == other.old_literal } @@ -145,8 +224,8 @@ fn apply_to_target( let _guard = bun_ast::expr::Disabler::scope(); let arena = &manager.ast_arena; for edit in edits { - match &edit.catalog { - None => { + match &edit.site { + EditSite::Dependencies => { for group in DEPENDENCY_GROUPS { let Some(mut query) = root.as_property(group) else { continue; @@ -154,12 +233,19 @@ fn apply_to_target( rewrite_property(arena, &mut query.expr, edit); } } - Some(catalog) => for_each_catalog_object(&root, |catalog_name, mut object| { - if CatalogMap::same_name(catalog_name, catalog) { - rewrite_property(arena, &mut object, edit); - } - Ok(()) - })?, + EditSite::Catalog(catalog) => { + for_each_catalog_object(&root, |catalog_name, mut object| { + if CatalogMap::same_name(catalog_name, catalog) { + rewrite_property(arena, &mut object, edit); + } + Ok(()) + })?; + } + EditSite::Override(selector) => { + for_each_override_value(&root, &edit.key, selector, |value| { + rewrite_value(arena, value, edit); + }); + } } } } @@ -175,19 +261,95 @@ fn rewrite_property(arena: &bun_alloc::Arena, object: &mut Expr, edit: &PackageJ let Some(key) = prop.key.as_ref().and_then(Expr::as_utf8_string_literal) else { continue; }; - if key != &*edit.key { - continue; + if key == &*edit.key { + rewrite_value(arena, &mut prop.value, edit); } - let Some(value) = prop.value.as_ref().and_then(Expr::as_utf8_string_literal) else { + } +} + +fn rewrite_value(arena: &bun_alloc::Arena, value: &mut Option, edit: &PackageJsonEdit) { + let Some(literal) = value.as_ref().and_then(Expr::as_utf8_string_literal) else { + return; + }; + if strings::trim(literal, &strings::WHITESPACE_CHARS) != &*edit.old_literal { + return; + } + *value = Some(Expr::allocate( + arena, + E::EString::init(arena.alloc_slice_copy(&edit.new_literal)), + bun_ast::Loc::EMPTY, + )); +} + +pub(super) fn root_package_json(manager: &mut PackageManager) -> Expr { + let target = target_for(manager, 0).expect("root target"); + fetch_entry_root(manager, &target) +} + +/// The string the root package.json currently gives `name`'s rule `selector`; when several entries spell the same rule, the last one, which is the one `OverrideMap` keeps. +pub(super) fn override_literal( + root: &Expr, + name: &[u8], + selector: &OverrideSelector, +) -> Option> { + let mut literal: Option> = None; + for_each_override_value(root, name, selector, |value| { + if let Some(text) = value.as_ref().and_then(Expr::as_utf8_string_literal) { + literal = Some(Box::from(strings::trim(text, &strings::WHITESPACE_CHARS))); + } + }); + literal +} + +/// Visits, in file order, the value of every entry declaring `selector` for `name`, whichever of the accepted key spellings (`a>b`, `a/b`, `**/b`, `{"a": {"b": ..}}`, `{"b": {".": ..}}`) each uses. +/// +/// Walks both `overrides` and `resolutions`, like `OverrideMap::parse_append`; npm's `"parent": { "child": .. }` form only counts in `overrides`, where the parser accepts it. +fn for_each_override_value( + root: &Expr, + name: &[u8], + selector: &OverrideSelector, + mut f: impl FnMut(&mut Option), +) { + for (field, nested) in [(b"overrides".as_slice(), true), (b"resolutions", false)] { + let Some(mut rules) = root.get(field) else { continue; }; - if strings::trim(value, &strings::WHITESPACE_CHARS) != &*edit.old_literal { + let Some(object) = rules.data.e_object_mut() else { continue; + }; + for prop in object.properties.slice_mut() { + let Some(key) = prop.key.as_ref().and_then(Expr::as_utf8_string_literal) else { + continue; + }; + if !(nested && prop.value.as_ref().is_some_and(|value| value.is_object())) { + if parse_selector(key).is_ok_and(|r| selector.matches(name, r.parent, r.target)) { + f(&mut prop.value); + } + continue; + } + let Ok(parent) = parse_package_segment(key) else { + continue; + }; + let Some(group) = prop.value.as_mut().and_then(|v| v.data.e_object_mut()) else { + continue; + }; + for child in group.properties.slice_mut() { + let Some(child_key) = child.key.as_ref().and_then(Expr::as_utf8_string_literal) + else { + continue; + }; + let (rule_parent, target) = match parse_selector(child_key) { + _ if child_key == b"." => (None, parent), + Ok(Selector { + parent: None, + target, + }) => (Some(parent), target), + _ => continue, + }; + if selector.matches(name, rule_parent, target) { + f(&mut child.value); + } + } } - prop.value = Some(Expr::allocate( - arena, - E::EString::init(arena.alloc_slice_copy(&edit.new_literal)), - bun_ast::Loc::EMPTY, - )); } } diff --git a/src/install/auto_installer.rs b/src/install/auto_installer.rs index 774a05069b6f..47d347cdd3fe 100644 --- a/src/install/auto_installer.rs +++ b/src/install/auto_installer.rs @@ -170,9 +170,10 @@ impl hooks::AutoInstaller for PackageManager { &self, name: &[u8], version: &hooks::DependencyVersion, + version_buf: &[u8], ) -> Option { self.lockfile - .resolve_package_from_name_and_version(name, version) + .resolve_package_from_name_and_version(name, version, version_buf) } fn lockfile_legacy_package_to_dependency_id( @@ -434,6 +435,31 @@ impl hooks::AutoInstaller for PackageManager { } } +// ─── Dependency parsing without a manager (resolver → install link-time hook) ── +// +// `bun_resolver::PackageJSON::parse` reads the project's package.json before the +// first bare import creates the `PackageManager` (see `Resolver::parse_dependency`). +// `parse` only uses the manager to record `npm:` aliases, and +// `lockfile_append_from_package_json` records them when these dependencies are +// cloned into the lockfile, so nothing is lost by parsing without one. +#[unsafe(no_mangle)] +fn __bun_resolver_parse_dependency( + name: SemverString, + name_hash: hooks::PackageNameHash, + version: &[u8], + sliced: &SlicedString, + log: Option<&mut bun_ast::Log>, +) -> Option { + dependency::parse( + name, + name_hash, + version, + sliced, + log, + None::<&mut PackageManager>, + ) +} + // ─── Lazy factory (resolver → install link-time hook) ───────────────────── // // `bun_resolver` cannot name `PackageManager` (it would create a dep cycle), diff --git a/src/install/bin.rs b/src/install/bin.rs index 7bb19bf33bbf..48fe1b06cabc 100644 --- a/src/install/bin.rs +++ b/src/install/bin.rs @@ -12,7 +12,7 @@ use bun_paths::MAX_PATH_BYTES; #[cfg(windows)] use bun_paths::WPathBuffer; use bun_paths::platform::Auto as PlatformAuto; -use bun_paths::resolve_path; +use bun_paths::resolve_path::{self, join_abs_string_buf_z_checked as join_z_checked}; use bun_paths::strings; use bun_paths::{self as path, AbsPath, PathBuffer, SEP}; use bun_semver::{ExternalString, String}; @@ -755,7 +755,7 @@ fn normalized_bin_name(name: &[u8]) -> &[u8] { /// verbatim from package.json, so without this check a malicious package could /// point a bin link at (and chmod) an arbitrary file on disk (the bug class /// npm fixed as CVE-2019-16775). -pub(crate) fn bin_target_escapes_package_dir(target: &[u8]) -> bool { +pub fn bin_target_escapes_package_dir(target: &[u8]) -> bool { if path::is_absolute(target) { return true; } @@ -925,29 +925,34 @@ impl<'a> Linker<'a> { bun_core::analytics::Features::binlinks_inc(); + // Only this bin's outcome decides whether the link just made is kept. + let prior_err = self.err.take(); #[cfg(not(windows))] { self.create_symlink(abs_target, abs_dest, global); } #[cfg(windows)] { - let target = match sys::File::openat(Fd::cwd(), abs_target, sys::O::RDONLY, 0) { - Ok(f) => f, + match sys::File::openat(Fd::cwd(), abs_target, sys::O::RDONLY, 0) { + Ok(target) => self.create_windows_shim(&target, abs_target, abs_dest, global), Err(err) => { let err: crate::Error = err.into(); + // ignore directories, creating a shim for one won't do anything if err != crate::Error::Sys(bun_errno::SystemErrno::EISDIR) { - // ignore directories, creating a shim for one won't do anything self.err = Some(err); } - return; } - }; - self.create_windows_shim(&target, abs_target, abs_dest, global); + } } + let err = self.err; + self.err = prior_err.or(err); - if self.err.is_some() { + if err.is_some() { // cleanup on error just in case Self::unlink_bin_or_shim(abs_dest); + if let Some(seen) = self.seen.as_deref_mut() { + seen.remove(abs_dest.as_bytes()); + } return; } @@ -1042,11 +1047,11 @@ impl<'a> Linker<'a> { // Create temporary file path let mut tmppath_buf = [0u8; MAX_PATH_BYTES]; - let tmppath = resolve_path::join_abs_string_buf_z::( - dir_path, - &mut tmppath_buf, - &[tmpname.as_bytes()], - ); + let Some(tmppath) = + join_z_checked::(dir_path, &mut tmppath_buf, &[tmpname.as_bytes()]) + else { + return; + }; let mut needs_unlink = true; let unlink_guard = scopeguard::guard(&mut needs_unlink, |needs_unlink| { if *needs_unlink { @@ -1179,12 +1184,16 @@ impl<'a> Linker<'a> { resolve_path::dirname::(abs_dest.as_bytes()), abs_target.as_bytes(), ); - debug_assert!(strings::has_prefix(rel_target.as_bytes(), b"..\\")); - let rel_target_w = strings::to_w_path_normalized( - target_buf.as_mut_slice(), - &rel_target.as_bytes()[b"..\\".len()..], - ); + // The shim resolves `bin_path` against its directory's parent, hence no `..\`. Global + // installs can target another drive or the bin directory itself: store those absolute. + let (bin_path, is_absolute_target): (&[u8], bool) = + match strings::without_prefix_if_possible_comptime(rel_target.as_bytes(), b"..\\") { + Some(rel_to_parent) => (rel_to_parent, false), + None => (abs_target.as_bytes(), true), + }; + + let bin_path_w = strings::to_w_path_normalized(target_buf.as_mut_slice(), bin_path); let shebang = 'shebang: { let first_content_chunk: Option<&[u8]> = 'contents: { @@ -1201,7 +1210,7 @@ impl<'a> Linker<'a> { }; if let Some(chunk) = first_content_chunk { - match WinShimShebang::parse(chunk, rel_target_w) { + match WinShimShebang::parse(chunk, bin_path_w) { Ok(s) => break 'shebang s, Err(_) => { self.err = Some(crate::Error::InvalidBinCount); @@ -1209,12 +1218,13 @@ impl<'a> Linker<'a> { } } } else { - break 'shebang WinShimShebang::parse_from_bin_path(rel_target_w); + break 'shebang WinShimShebang::parse_from_bin_path(bin_path_w); } }; let shim = WinBinLinkingShim { - bin_path: rel_target_w, + bin_path: bin_path_w, + is_absolute_target, shebang, }; @@ -1263,10 +1273,12 @@ impl<'a> Linker<'a> { // so each return path calls `Self::chmod_on_ok` explicitly instead. let abs_dest_dir = resolve_path::dirname::(abs_dest.as_bytes()); - let rel_target = - resolve_path::relative_buf_z(self.rel_buf, abs_dest_dir, abs_target.as_bytes()); - - debug_assert!(strings::has_prefix(rel_target.as_bytes(), b"..")); + // Need not start with `..` (a global install may nest the package inside the bin + // directory), and one `..` per component of `abs_dest_dir` can outgrow `abs_target`. + let rel_bound = abs_target.len() + 3 * (strings::count_char(abs_dest_dir, SEP) + 1) + 2; + let mut rel_spill = Vec::new(); + let rel_buf = resolve_path::buf_or_spill(self.rel_buf, &mut rel_spill, rel_bound); + let rel_target = resolve_path::relative_buf_z(rel_buf, abs_dest_dir, abs_target.as_bytes()); match sys::symlink_running_executable(rel_target, abs_dest) { sys::Result::Err(err) => { @@ -1464,59 +1476,45 @@ impl<'a> Linker<'a> { /// /// Falls through to (1) when nothing exists so the existing /// `skipped_due_to_missing_bin` retry-without-redirect path still fires. - // `is_native_binlink_redirect()` is hoisted to a parameter so the - // caller can drop its `&self` borrow before mutably calling - // `link_bin_or_create_shim`. Result borrows the threadlocal join buffer - // (lifetime tied to `package_dir` per `join_abs_string_z`'s signature). + /// `None`: the path does not fit `buf`; callers treat that as a missing bin. fn resolve_bin_target<'b>( is_native_binlink_redirect: bool, - package_dir: &'b [u8], + package_dir: &[u8], target: &[u8], bin_name: &[u8], - ) -> &'b ZStr { + buf: &'b mut [u8], + ) -> Option<&'b ZStr> { // A trailing separator would make `lchmod` follow a symlinked target; npm drops it too. let target = strings::without_trailing_slash(target); - let primary = resolve_path::join_abs_string_z::(package_dir, &[target]); if !is_native_binlink_redirect { - return primary; - } - - if sys::exists(primary.as_bytes()) { - return primary; - } - - if !bin_name.is_empty() { - let at_root = resolve_path::join_abs_string_z::(package_dir, &[bin_name]); - if sys::exists(at_root.as_bytes()) { - return at_root; - } + return join_z_checked::(package_dir, buf, &[target]); } let target_basename = path::basename(target); - if !target_basename.is_empty() && target_basename.len() != target.len() { - let at_root = - resolve_path::join_abs_string_z::(package_dir, &[target_basename]); - if sys::exists(at_root.as_bytes()) { - return at_root; + let in_subdir = target_basename.len() != target.len(); + let exe_name: Vec = + if !bin_name.is_empty() && !strings::has_suffix_comptime(bin_name, b".exe") { + [bin_name, b".exe"].concat() + } else { + Vec::new() + }; + let at_root = if in_subdir { target_basename } else { b"" }; + for candidate in [target, bin_name, at_root, &exe_name] { + if candidate.is_empty() { + continue; } - } - - if !bin_name.is_empty() && !strings::has_suffix_comptime(bin_name, b".exe") { - let mut exe_name = Vec::with_capacity(bin_name.len() + b".exe".len()); - exe_name.extend_from_slice(bin_name); - exe_name.extend_from_slice(b".exe"); - let at_root = - resolve_path::join_abs_string_z::(package_dir, &[&exe_name]); - if sys::exists(at_root.as_bytes()) { - return at_root; + let found = join_z_checked::(package_dir, buf, &[candidate]) + .filter(|abs| sys::exists_z(abs)); + if let Some(len) = found.map(ZStr::len) { + return Some(ZStr::from_buf(buf, len)); } } - // Nothing found; return the primary so `linkBinOrCreateShim` sets + // Nothing found; return the primary so `link_bin_or_create_shim` sets // `skipped_due_to_missing_bin` and the caller retries without the // redirect. - resolve_path::join_abs_string_z::(package_dir, &[target]) + join_z_checked::(package_dir, buf, &[target]) } /// uses `self.abs_target_buf` @@ -1588,17 +1586,13 @@ impl<'a> Linker<'a> { debug_assert!(self.bin.tag != Tag::None); - // `link_bin_or_create_shim(&mut self, ..)` - // is called while `abs_target` / `abs_dest` borrow `self.abs_target_buf` - // / `self.abs_dest_buf`. `link_bin_or_create_shim` never reads or writes - // those two buffers (it only touches `rel_buf`, `node_modules_path`, `seen`, `err`, - // `skipped_due_to_missing_bin`). Detach the `abs_dest` borrow via a raw - // pointer so borrowck allows the disjoint access; the SAFETY invariant - // is that `abs_dest_buf` is not aliased mutably for the lifetime of the - // detached slice. `package_dir` (`abs_target_buf[0..package_dir_len]`) - // is re-derived inside each arm so no detached borrow is needed for it. + // SAFETY (`from_raw` below): `link_bin_or_create_shim` never touches `abs_dest_buf`. let abs_dest_buf_ptr: *mut u8 = self.abs_dest_buf.as_mut_ptr(); + // `abs_target_buf` holds `package_dir`, the join input, so the output needs its own buffer. + let mut resolved_target_pool_buf = path::path_buffer_pool::get(); + let resolved_target_buf = resolved_target_pool_buf.as_mut_slice(); + // SAFETY: tag determines the active union field unsafe { match self.bin.tag { @@ -1616,20 +1610,15 @@ impl<'a> Linker<'a> { Dependency::unscoped_package_name(self.package_name.slice()); // for normalizing `target` - let abs_target: &ZStr = { - let package_dir = &self.abs_target_buf[0..package_dir_len]; - let r = Self::resolve_bin_target( - is_redirect, - package_dir, - target, - unscoped_package_name, - ); - // SAFETY: `resolve_bin_target` writes into the thread-local - // `PARSER_JOIN_INPUT_BUFFER` (via `join_abs_string_z`); the - // returned slice does not actually borrow `self` or - // `package_dir`. Detach the lifetime so `self` can be - // re-borrowed mutably below. - ZStr::from_raw(r.as_bytes().as_ptr(), r.len()) + let Some(abs_target) = Self::resolve_bin_target( + is_redirect, + &self.abs_target_buf[0..package_dir_len], + target, + unscoped_package_name, + resolved_target_buf, + ) else { + self.skipped_due_to_missing_bin = true; + return; }; if unscoped_package_name.len() @@ -1672,16 +1661,15 @@ impl<'a> Linker<'a> { } // for normalizing `target` - let abs_target: &ZStr = { - let package_dir = &self.abs_target_buf[0..package_dir_len]; - let r = Self::resolve_bin_target( - is_redirect, - package_dir, - target, - normalized_name, - ); - // SAFETY: thread-local buffer; see Tag::File above. - ZStr::from_raw(r.as_bytes().as_ptr(), r.len()) + let Some(abs_target) = Self::resolve_bin_target( + is_redirect, + &self.abs_target_buf[0..package_dir_len], + target, + normalized_name, + resolved_target_buf, + ) else { + self.skipped_due_to_missing_bin = true; + return; }; self.abs_dest_buf[dest_off..dest_off + normalized_name.len()] @@ -1728,16 +1716,16 @@ impl<'a> Linker<'a> { return; } - let abs_target: &ZStr = { - let package_dir = &self.abs_target_buf[0..package_dir_len]; - let r = Self::resolve_bin_target( - is_redirect, - package_dir, - bin_target, - normalized_bin_dest, - ); - // SAFETY: thread-local buffer; see Tag::File above. - ZStr::from_raw(r.as_bytes().as_ptr(), r.len()) + let Some(abs_target) = Self::resolve_bin_target( + is_redirect, + &self.abs_target_buf[0..package_dir_len], + bin_target, + normalized_bin_dest, + resolved_target_buf, + ) else { + self.skipped_due_to_missing_bin = true; + i += 2; + continue; }; dest_off = abs_dest_dir_end; @@ -1766,15 +1754,13 @@ impl<'a> Linker<'a> { return; } // for normalizing `target` - let abs_target_dir: &ZStr = { - let package_dir = &self.abs_target_buf[0..package_dir_len]; - let r = - resolve_path::join_abs_string_z::(package_dir, &[target]); - // SAFETY: `join_abs_string_z` writes into the thread-local - // `PARSER_JOIN_INPUT_BUFFER`; result does not borrow - // `package_dir`. Detached so `abs_target_buf` can be - // reused inside the loop body (see the SAFETY note below). - ZStr::from_raw(r.as_bytes().as_ptr(), r.len()) + let Some(abs_target_dir) = join_z_checked::( + &self.abs_target_buf[0..package_dir_len], + resolved_target_buf, + &[target], + ) else { + self.err = Some(crate::Error::Sys(bun_errno::SystemErrno::ENAMETOOLONG)); + return; }; let target_dir = match sys::open_dir_absolute(abs_target_dir.as_bytes()) { @@ -1800,13 +1786,16 @@ impl<'a> Linker<'a> { match entry.kind { sys::EntryKind::SymLink | sys::EntryKind::File => { let entry_name = entry.name.slice_u8(); - // `self.abs_target_buf` is available now because `path::join_abs_string_z` copied everything into `parse_join_input_buffer` + // `package_dir` is no longer needed, so `abs_target_buf` is free. let abs_target: &ZStr = { - let r = resolve_path::join_abs_string_buf_z::( + let Some(r) = join_z_checked::( abs_target_dir.as_bytes(), self.abs_target_buf, &[entry_name], - ); + ) else { + self.skipped_due_to_missing_bin = true; + continue; + }; // SAFETY: result lives in `self.abs_target_buf`, which // `link_bin_or_create_shim` does not write to (only // `rel_buf`/`node_modules_path`/`seen`/`err`/ @@ -1847,11 +1836,6 @@ impl<'a> Linker<'a> { debug_assert!(self.bin.tag != Tag::None); - // see `link()` — detach abs_target_buf borrow via raw ptr. - let abs_target_buf_ptr: *const u8 = self.abs_target_buf.as_ptr(); - // SAFETY: abs_target_buf is not written between here and use. - let package_dir = unsafe { bun_core::ffi::slice(abs_target_buf_ptr, package_dir_len) }; - // SAFETY: tag determines the active union field unsafe { match self.bin.tag { @@ -1936,8 +1920,15 @@ impl<'a> Linker<'a> { return; } - let abs_target_dir = - resolve_path::join_abs_string_z::(package_dir, &[target]); + let mut abs_target_dir_buf = path::path_buffer_pool::get(); + let Some(abs_target_dir) = join_z_checked::( + &self.abs_target_buf[0..package_dir_len], + abs_target_dir_buf.as_mut_slice(), + &[target], + ) else { + self.err = Some(crate::Error::Sys(bun_errno::SystemErrno::ENAMETOOLONG)); + return; + }; let target_dir = match sys::open_dir_absolute(abs_target_dir.as_bytes()) { Ok(d) => d, diff --git a/src/install/dedupe.rs b/src/install/dedupe.rs index 84185e1302d4..2c61109395be 100644 --- a/src/install/dedupe.rs +++ b/src/install/dedupe.rs @@ -6,6 +6,7 @@ use bun_collections::bit_set::Range; use bun_collections::{DynamicBitSet, index_sort}; use bun_core::{Global, Output, UnwrapOrOom as _, strings}; +use crate::lockfile::override_map::OverrideRule; use crate::lockfile::package::PackageColumns as _; use crate::lockfile::{LoadResult, Lockfile, Package, PackageIndexEntry}; use crate::package_manager::Options::{Enable, LogLevel}; @@ -308,6 +309,50 @@ pub(crate) fn label(lockfile: &Lockfile, id: PackageID) -> Vec { label } +// Biggest group first, dependents (at most 3) in name order, versioned only when the name alone is ambiguous. +fn format_wanted( + lockfile: &Lockfile, + group_of: &[u32], + mut groups: Vec<(Box<[u8]>, Vec)>, +) -> Vec { + let buf = lockfile.buffers.string_bytes.as_slice(); + let names = lockfile.packages.items_name(); + index_sort::sort_vec_by(&mut groups, |(range_a, owners_a), (range_b, owners_b)| { + owners_b + .len() + .cmp(&owners_a.len()) + .then_with(|| strings::order(range_a, range_b)) + }); + groups + .into_iter() + .map(|(range, mut owners)| { + index_sort::sort_vec_by(&mut owners, |&a, &b| { + strings::order(names[a as usize].slice(buf), names[b as usize].slice(buf)) + .then(a.cmp(&b)) + }); + let shown = owners.len().min(3); + let mut dependents = Vec::new(); + for (i, &owner) in owners[..shown].iter().enumerate() { + if i > 0 { + dependents.extend_from_slice(b", "); + } + if group_of[owner as usize] == u32::MAX { + dependents.extend_from_slice(names[owner as usize].slice(buf)); + } else { + dependents.extend_from_slice(&label(lockfile, owner)); + } + } + if owners.len() > shown { + let _ = write!(dependents, " +{} more", owners.len() - shown); + } + Wanted { + range, + dependents: dependents.into_boxed_slice(), + } + }) + .collect() +} + fn order_by_name_then_version(lockfile: &Lockfile, a: PackageID, b: PackageID) -> Ordering { let buf = lockfile.buffers.string_bytes.as_slice(); let names = lockfile.packages.items_name(); @@ -325,8 +370,21 @@ fn sort_by_name_then_version(lockfile: &Lockfile, ids: &mut [PackageID]) { index_sort::sort_indices(ids, &mut |a, b| order_by_name_then_version(lockfile, a, b)); } -// (name, removed version, surviving version(s) its dependents now resolve to) -type Row = (Box<[u8]>, Box<[u8]>, Box<[u8]>); +// One `wanted by ` line under a row (`--why`). +struct Wanted { + range: Box<[u8]>, + dependents: Box<[u8]>, +} + +struct Row { + name: Box<[u8]>, + from: Box<[u8]>, + /// Surviving version(s); empty when `from` is dropped outright. + to: Box<[u8]>, + /// Every survivor is a lower major than `from`. + downgrade: bool, + wanted: Vec, +} #[derive(Default)] pub(crate) struct Report { @@ -339,7 +397,7 @@ fn plural(n: usize) -> &'static str { if n == 1 { "" } else { "s" } } -fn dedupe_lockfile(lockfile: &mut Lockfile) -> Report { +fn dedupe_lockfile(lockfile: &mut Lockfile, why: bool) -> Report { let buf = lockfile.buffers.string_bytes.as_slice(); let pkg_res = lockfile.packages.items_resolution(); let checked = pkg_res.len(); @@ -536,12 +594,59 @@ fn dedupe_lockfile(lockfile: &mut Lockfile) -> Report { } let names = lockfile.packages.items_name(); + + // Per row: dependents whose edges targeted the removed version or one of its survivors, by requested range. + let mut wanted: Vec, Vec)>> = vec![Vec::new(); removed.len()]; + if why { + let deps = lockfile.buffers.dependencies.as_slice(); + // A version explains the rows it was removed by and the rows whose edges moved onto it. + let mut explains: Vec> = vec![Vec::new(); pkg_res.len()]; + for (i, &id) in removed.iter().enumerate() { + explains[id as usize].push(i as u32); + } + for (i, survivors) in targets.iter().enumerate() { + for &s in survivors { + explains[s as usize].push(i as u32); + } + } + // Owners come from the pre-dedupe tree so a dropped version's (themselves removed) dependents still show. + for (owner, slice) in lockfile.packages.items_dependencies().iter().enumerate() { + if !initial.is_set(owner) { + continue; + } + for dep_id in slice.begin() as usize..slice.end() as usize { + let Some(rows) = explains.get(original[dep_id] as usize) else { + continue; + }; + let dep = &deps[dep_id]; + let literal = effective_version(lockfile, dep_id as DependencyID, dep) + .map(|version| version.literal) + .unwrap_or(dep.version.literal); + let range = literal.slice(buf); + for &r in rows { + let groups = &mut wanted[r as usize]; + match groups.iter_mut().find(|(g, _)| strings::eql(g, range)) { + Some((_, owners)) => { + // Two aliases in one package can produce identical edges; list the owner once. + if !owners.contains(&(owner as PackageID)) { + owners.push(owner as PackageID); + } + } + None => groups.push((Box::from(range), vec![owner as PackageID])), + } + } + } + } + } + let rows: Vec = removed .iter() .zip(&targets) - .map(|(&id, moved_to)| { + .zip(wanted) + .map(|((&id, moved_to), wanted)| { + let from_version = pkg_res[id as usize].npm().version; let mut from: Vec = Vec::new(); - let _ = write!(from, "{}", pkg_res[id as usize].npm().version.fmt(buf)); + let _ = write!(from, "{}", from_version.fmt(buf)); let mut survivors: Vec = moved_to.clone(); index_sort::sort_vec_by(&mut survivors, |&a, &b| { pkg_res[a as usize] @@ -556,11 +661,16 @@ fn dedupe_lockfile(lockfile: &mut Lockfile) -> Report { } let _ = write!(to, "{}", pkg_res[c as usize].npm().version.fmt(buf)); } - ( - Box::from(names[id as usize].slice(buf)), - from.into_boxed_slice(), - to.into_boxed_slice(), - ) + let downgrade = survivors + .last() + .is_some_and(|&c| pkg_res[c as usize].npm().version.major < from_version.major); + Row { + name: Box::from(names[id as usize].slice(buf)), + from: from.into_boxed_slice(), + to: to.into_boxed_slice(), + downgrade, + wanted: format_wanted(lockfile, &group_of, wanted), + } }) .collect(); @@ -581,21 +691,27 @@ pub(crate) fn effective_npm_range( .filter(|version| version.tag == DependencyVersionTag::Npm) } +/// The override rule the resolver applies to this edge; `workspace:` and `npm:` alias edges are never overridden (PackageManagerEnqueue.rs). +pub(crate) fn applied_override<'a>( + lockfile: &'a Lockfile, + dep_id: DependencyID, + dep: &Dependency, +) -> Option> { + if dep.behavior.is_workspace() + || (dep.version.tag == DependencyVersionTag::Npm && dep.version.npm().is_alias) + { + return None; + } + lockfile.overrides.lookup(lockfile, dep_id, dep.name_hash) +} + pub(crate) fn effective_version( lockfile: &Lockfile, dep_id: DependencyID, dep: &Dependency, ) -> Option { - let mut version = if dep.behavior.is_workspace() - || (dep.version.tag == DependencyVersionTag::Npm && dep.version.npm().is_alias) - { - dep.version.clone() - } else { - lockfile - .overrides - .get(lockfile, dep_id, dep.name_hash) - .unwrap_or_else(|| dep.version.clone()) - }; + let mut version = applied_override(lockfile, dep_id, dep) + .map_or_else(|| dep.version.clone(), |rule| rule.version().clone()); if version.tag == DependencyVersionTag::Catalog { version = lockfile .catalogs @@ -605,7 +721,8 @@ pub(crate) fn effective_version( Some(version) } -// Optional-peer edges are followed too: with an in-sync package.json `clean` runs with `keep_optional_peer_targets`. +// Optional-peer edges are followed too: `clean` keeps a target the loaded lockfile held through them alone +// (`Lockfile::held_at_load`), and a version that survives here keeps every edge it had, so anything else they reach stays held. fn reachable(lockfile: &Lockfile, resolutions: &[PackageID]) -> DynamicBitSet { crate::lockfile::reachable::packages( lockfile, @@ -729,7 +846,7 @@ fn report_already_deduplicated(manager: &PackageManager, report: &Report) -> ! { bun_core::pretty!("\n"); } bun_core::pretty!( - "🎉 No duplicates — checked {} package{}, every one already resolves to a single version ", + "🎉 No duplicates — checked {} package{} in bun.lock, every one already resolves to a single version ", report.checked, plural(report.checked) ); @@ -757,7 +874,7 @@ fn print_would_remove(manager: &PackageManager, report: &Report) { } let n = report.rows.len(); bun_core::pretty!( - "\n{} duplicate version{} can be removed (checked {} package{}) ", + "\n{} duplicate version{} can be removed (checked {} package{} in bun.lock) ", n, plural(n), report.checked, @@ -773,22 +890,43 @@ fn print_rows(report: &Report) { } else { ("~", "->") }; - for (name, from, to) in &report.rows { - if to.is_empty() { + for row in &report.rows { + if row.to.is_empty() { bun_core::prettyln!( - "{} {} {}", + "{} {} {} {} (removed)", glyph, - BStr::new(name), - BStr::new(from) + BStr::new(&row.name), + BStr::new(&row.from), + arrow + ); + } else if row.downgrade { + bun_core::prettyln!( + "{} {} {} {} {} (downgrade)", + glyph, + BStr::new(&row.name), + BStr::new(&row.from), + arrow, + BStr::new(&row.to) ); } else { bun_core::prettyln!( "{} {} {} {} {}", glyph, - BStr::new(name), - BStr::new(from), + BStr::new(&row.name), + BStr::new(&row.from), arrow, - BStr::new(to) + BStr::new(&row.to) + ); + } + let width = row.wanted.iter().map(|w| w.range.len()).max().unwrap_or(0); + for w in &row.wanted { + let mut range = Vec::with_capacity(width); + range.extend_from_slice(&w.range); + range.resize(width, b' '); + bun_core::prettyln!( + " wanted {} by {}", + BStr::new(&range), + BStr::new(&w.dependents) ); } } @@ -817,7 +955,7 @@ pub(crate) fn print_dedupe_summary(manager: &PackageManager, installed: u32, sta ); } bun_core::pretty!( - " (checked {} package{}) ", + " (checked {} package{} in bun.lock) ", report.checked, plural(report.checked) ); @@ -833,7 +971,7 @@ pub fn dedupe_after_differ(manager: &mut PackageManager) { refuse_out_of_date(manager); } - let report = dedupe_lockfile(&mut manager.lockfile); + let report = dedupe_lockfile(&mut manager.lockfile, manager.options.why); if report.rows.is_empty() { report_already_deduplicated(manager, &report); } diff --git a/src/install/dependency.rs b/src/install/dependency.rs index 2f5b38f9b895..575fe41046fb 100644 --- a/src/install/dependency.rs +++ b/src/install/dependency.rs @@ -417,7 +417,7 @@ pub(crate) fn is_scp_like_path(dependency: &[u8]) -> bool { /// /// This also checks for a github url that ends with ".tar.gz" #[inline] -fn is_github_tarball_path(dependency: &[u8]) -> bool { +pub(crate) fn is_github_tarball_path(dependency: &[u8]) -> bool { if is_tarball(dependency) { return true; } @@ -557,9 +557,9 @@ pub fn is_scoped_package_name(name: &[u8]) -> Result { /// A dependency name/alias becomes a directory under `node_modules/`. Names /// come from untrusted `package.json` / manifest keys, so reject anything that -/// could resolve outside that directory. `@scope/name` stays valid. +/// could resolve outside that directory, and terminal control characters. `@scope/name` stays valid. pub(crate) fn is_safe_install_folder_name(name: &[u8]) -> bool { - if name.is_empty() { + if name.is_empty() || contains_control_character(name) { return false; } @@ -567,7 +567,7 @@ pub(crate) fn is_safe_install_folder_name(name: &[u8]) -> bool { if component.is_empty() || component == b"." || component == b".." { return false; } - if strings::contains_any(component, b"\\:\0") { + if strings::contains_any(component, b"\\:") { return false; } } @@ -575,6 +575,39 @@ pub(crate) fn is_safe_install_folder_name(name: &[u8]) -> bool { true } +/// C0 controls and DEL, plus UTF-8 encoded C1 controls (`C2 80`..`C2 9F`, +/// U+0080..=U+009F), which terminals interpret too. +pub(crate) fn contains_control_character(name: &[u8]) -> bool { + name.iter().enumerate().any(|(i, &byte)| { + byte.is_ascii_control() + || (byte == 0xC2 + && name + .get(i + 1) + .is_some_and(|next| (0x80..=0x9F).contains(next))) + }) +} + +/// A name bun.lock's `"@"` keys can round-trip: a safe folder name with no `@` past the scope marker. +pub(crate) fn is_safe_lockfile_package_name(name: &[u8]) -> bool { + is_safe_install_folder_name(name) && !strings::contains_char(&name[1..], b'@') +} + +/// Name for a package whose package.json has none: the last component of where it came from +/// (`../pkgs/foo`, `https://host/foo.tgz?token=x`, `https://host/user/foo` all become `foo`). +pub(crate) fn fallback_package_name(location: &[u8]) -> &[u8] { + let without_query = strings::split(location, b"?").next().unwrap_or(location); + let basename = bun_paths::basename(without_query); + let name = strings::without_suffix_comptime( + strings::without_suffix_comptime(basename, b".tgz"), + b".tar.gz", + ); + if is_safe_lockfile_package_name(name) { + name + } else { + b"unnamed-package" + } +} + /// assumes version is valid pub fn without_build_tag(version: &[u8]) -> &[u8] { if let Some(plus) = strings::index_of_char(version, b'+') { @@ -871,7 +904,11 @@ impl TagExt for Tag { } } b'+' => { - if url.starts_with(b"+ssh:") || url.starts_with(b"+file:") { + // bun.lock writes a `git://` dependency's resolution as `git+git:`. + if url.starts_with(b"+ssh:") + || url.starts_with(b"+file:") + || url.starts_with(b"+git:") + { return Tag::Git; } if url.starts_with(b"+http") { @@ -1154,6 +1191,12 @@ pub(crate) fn is_windows_abs_path_with_leading_slashes(dep: &[u8]) -> Option<&[u None } +/// Literals may carry leading whitespace; classify and parse these bytes, not the raw literal. +#[inline] +pub fn trim_literal(literal: &[u8]) -> &[u8] { + strings::trim_left(literal, b" \t\n\r") +} + #[inline] pub fn parse<'a, 'b>( alias: String, @@ -1163,7 +1206,7 @@ pub fn parse<'a, 'b>( log: impl Into>, manager: impl Into>, ) -> Option { - let dep = strings::trim_left(dependency, b" \t\n\r"); + let dep = trim_literal(dependency); parse_with_tag( alias, alias_hash.into(), @@ -1184,7 +1227,7 @@ pub(crate) fn parse_with_optional_tag<'a, 'b>( log: impl Into>, package_manager: impl Into>, ) -> Option { - let dep = strings::trim_left(dependency, b" \t\n\r"); + let dep = trim_literal(dependency); parse_with_tag( alias, alias_hash.into(), @@ -1207,6 +1250,8 @@ pub(crate) fn parse_with_tag( log_: Option<&mut bun_ast::Log>, package_manager: Option<&mut dyn NpmAliasRegistry>, ) -> Option { + // `to_version` (bun.lockb) and `clone_with_different_buffers` pass the stored literal untrimmed. + let dependency = trim_literal(dependency); match tag { Tag::Npm => { let mut input = dependency; @@ -1272,7 +1317,7 @@ pub(crate) fn parse_with_tag( Some(result) } Tag::DistTag => { - let mut tag_to_use = sliced.value(); + let mut tag_to_use = sliced.sub(dependency).value(); let actual = if dependency.starts_with(b"npm:") && dependency.len() > b"npm:".len() { // npm:@foo/bar@latest @@ -1445,7 +1490,7 @@ pub(crate) fn parse_with_tag( bun_ast::Loc::EMPTY, format_args!( "invalid or unsupported dependency \"{}\"", - bstr::BStr::new(dependency) + bun_core::fmt::escape_control_chars(dependency) ), ); } @@ -1457,7 +1502,7 @@ pub(crate) fn parse_with_tag( literal: sliced.value(), value: Value { tarball: TarballInfo { - uri: URI::Local(sliced.value()), + uri: URI::Local(sliced.sub(dependency).value()), package_name: String::default(), }, }, @@ -1566,7 +1611,10 @@ pub(crate) fn parse_with_tag( log.add_error_fmt( None, bun_ast::Loc::EMPTY, - format_args!("Unsupported protocol {}", bstr::BStr::new(dependency)), + format_args!( + "Unsupported protocol {}", + bun_core::fmt::escape_control_chars(dependency) + ), ); } return None; @@ -1574,7 +1622,7 @@ pub(crate) fn parse_with_tag( Some(Version { value: Value { - folder: sliced.value(), + folder: sliced.sub(dependency).value(), }, tag: Tag::Folder, literal: sliced.value(), @@ -1598,7 +1646,7 @@ pub(crate) fn parse_with_tag( Some(Version { value: Value { - symlink: sliced.value(), + symlink: sliced.sub(dependency).value(), }, tag: Tag::Symlink, literal: sliced.value(), diff --git a/src/install/error.rs b/src/install/error.rs index 6219d1ed0dc8..bf5bd11e7e38 100644 --- a/src/install/error.rs +++ b/src/install/error.rs @@ -70,6 +70,8 @@ pub enum Error { Failed, #[error("UnrecognizedDependencyFormat")] UnrecognizedDependencyFormat, + #[error("InvalidDependencyName")] + InvalidDependencyName, #[error("No global directory found")] NoGlobalDirectoryFound, #[error("InvalidPackageID")] @@ -176,8 +178,6 @@ pub enum Error { CorruptLockfile, #[error("Lockfile is missing resolution data")] LockfileIsMissingResolutionData, - #[error("MissingPackageName")] - MissingPackageName, #[error("GlobError")] GlobError, #[error("Invalid")] @@ -282,6 +282,7 @@ impl Error { Self::HTTPError => "HTTPError", Self::Failed => "Failed", Self::UnrecognizedDependencyFormat => "UnrecognizedDependencyFormat", + Self::InvalidDependencyName => "InvalidDependencyName", Self::NoGlobalDirectoryFound => "No global directory found", Self::InvalidPackageID => "InvalidPackageID", Self::PartialInstallFailed => "PartialInstallFailed", @@ -341,7 +342,6 @@ impl Error { } Self::CorruptLockfile => "CorruptLockfile", Self::LockfileIsMissingResolutionData => "Lockfile is missing resolution data", - Self::MissingPackageName => "MissingPackageName", Self::GlobError => "GlobError", Self::Invalid => "Invalid", Self::LockfileValidationFailedListIsImpossiblyLong => { diff --git a/src/install/extract_tarball.rs b/src/install/extract_tarball.rs index b8d3a8b091b5..6a156957fa54 100644 --- a/src/install/extract_tarball.rs +++ b/src/install/extract_tarball.rs @@ -57,28 +57,26 @@ impl ExtractTarball { return Err(crate::Error::IntegrityCheckFailed); } } - let mut result = self.extract(log, bytes)?; + let integrity = self.lockfile_integrity(|| Integrity::for_bytes(bytes)); + let mut result = self.extract(log, bytes, &integrity)?; + result.integrity = integrity; + Ok(result) + } - // Compute and store SHA-512 integrity hash for GitHub / URL / local tarballs - // so the lockfile can pin the exact tarball content. On subsequent installs - // the hash stored in the lockfile is forwarded via this.integrity and verified - // above, preventing a compromised server from silently swapping the tarball. + /// What the lockfile pins for a GitHub / URL / local tarball so later installs verify the + /// same bytes: the already pinned value, else `compute`d on first install. Settled before + /// extracting because a local tarball's cache entry is named after it. + pub(crate) fn lockfile_integrity(&self, compute: impl FnOnce() -> Integrity) -> Integrity { match self.resolution.tag { ResolutionTag::Github | ResolutionTag::RemoteTarball | ResolutionTag::LocalTarball => { if self.integrity.tag.is_supported() { - // Re-installing with an existing lockfile: integrity was already - // verified above, propagate the known value to ExtractData so that - // the lockfile keeps it on re-serialisation. - result.integrity = self.integrity; + self.integrity } else { - // First install (no integrity in the lockfile yet): compute it. - result.integrity = Integrity::for_bytes(bytes); + compute() } } - _ => {} + _ => Integrity::default(), } - - Ok(result) } } @@ -181,7 +179,38 @@ pub(crate) fn uses_streaming_extraction() -> bool { .unwrap_or(false) } +/// What `move_to_cache_directory` does with an existing cache folder of the same name. +/// Decided before extracting, while "pre-existing" and "published meanwhile" still differ. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum CachePublish { + /// npm and GitHub: swap ours in; leave the old one in the temp dir for concurrent readers. + Replace, + /// Tarball re-extracted over its own folder: swap ours in, delete the old one. + Supersede, + /// Tarball whose folder did not exist: one there now is a concurrent identical extraction. + KeepExisting, +} + impl ExtractTarball { + pub(crate) fn cache_publish(&self) -> CachePublish { + if !self.resolution.tag.is_tarball() { + return CachePublish::Replace; + } + let name_taken = TL_BUFS.with_borrow_mut(|bufs| { + let folder_name = directories::cached_tarball_folder_name_print( + &mut bufs.folder_name_buf, + self.url.slice(), + None, + ); + sys::exists_at_type(self.cache_dir, folder_name).is_ok() + }); + if name_taken { + CachePublish::Supersede + } else { + CachePublish::KeepExisting + } + } + /// Derive the display name and a filesystem-safe basename for this /// package. Shared by the buffered `extract()` path below and the /// streaming extractor in `TarballStream.rs` so both pick identical @@ -201,12 +230,20 @@ impl ExtractTarball { let basename: &[u8] = 'brk: { let mut tmp = name; if strings::has_prefix(tmp, b"https://") || strings::has_prefix(tmp, b"http://") { + // A URL name is the placeholder `bun add ` uses until package.json is read. + if let Some(i) = strings::index_of_any(tmp, b"?#") { + tmp = &tmp[0..i]; + } tmp = bun_paths::basename(tmp); if strings::ends_with(tmp, b".tgz") { tmp = &tmp[0..tmp.len() - 4]; } else if strings::ends_with(tmp, b".tar.gz") { tmp = &tmp[0..tmp.len() - 7]; } + if !bun_install::dependency::is_safe_install_folder_name(tmp) { + tmp = b"package"; + } + break 'brk tmp; } else if tmp[0] == b'@' { if let Some(i) = strings::index_of_char(tmp, b'/') { tmp = &tmp[i as usize + 1..]; @@ -225,7 +262,12 @@ impl ExtractTarball { (name, basename) } - fn extract(&self, log: &mut bun_ast::Log, tgz_bytes: &[u8]) -> Result { + fn extract( + &self, + log: &mut bun_ast::Log, + tgz_bytes: &[u8], + integrity: &Integrity, + ) -> Result { let _tracer = bun_core::perf::trace("ExtractTarball.extract"); let tmpdir = Dir::borrow(&self.temp_dir); @@ -247,7 +289,7 @@ impl ExtractTarball { bun_ast::Loc::EMPTY, format_args!( "Refusing to install package with invalid name \"{}\"", - bun_fmt::s(name), + bun_fmt::escape_control_chars(name), ), ); return Err(crate::Error::InstallFailed); @@ -256,6 +298,7 @@ impl ExtractTarball { let mut resolved: &'static [u8] = b""; let tmpname = FileSystem::tmpname(tmpname_suffix, &mut tmpname_buf.0, bun_core::fast_random())?; + let publish = self.cache_publish(); { let extract_destination = match bun_sys::make_path::make_open_path( tmpdir, @@ -439,12 +482,13 @@ impl ExtractTarball { } } - self.move_to_cache_directory(log, tmpname, name, basename, resolved) + self.move_to_cache_directory(log, tmpname, name, basename, resolved, integrity, publish) } /// Rename the freshly-extracted temp directory into the cache, read /// `package.json` if required, and build the `ExtractData` result. Shared - /// between the buffered and streaming extraction paths. + /// between the buffered and streaming extraction paths. `resolved` names a + /// GitHub cache entry, `integrity` a local tarball's. pub(crate) fn move_to_cache_directory( &self, log: &mut bun_ast::Log, @@ -452,6 +496,8 @@ impl ExtractTarball { name: &[u8], basename: &[u8], resolved: &[u8], + integrity: &Integrity, + publish: CachePublish, ) -> Result { let package_manager = self.package_manager.get(); @@ -466,7 +512,7 @@ impl ExtractTarball { bun_ast::Loc::EMPTY, format_args!( "Refusing to install package with invalid name \"{}\"", - bun_fmt::s(name), + bun_fmt::escape_control_chars(name), ), ); return Err(crate::Error::InstallFailed); @@ -500,14 +546,18 @@ impl ExtractTarball { ) .as_bytes() } - ResolutionTag::LocalTarball | ResolutionTag::RemoteTarball => { - directories::cached_tarball_folder_name_print( - &mut bufs.folder_name_buf, - self.url.slice(), - None, - ) - .as_bytes() - } + ResolutionTag::LocalTarball => directories::cached_local_tarball_folder_name_print( + &mut bufs.folder_name_buf, + integrity, + None, + ) + .as_bytes(), + ResolutionTag::RemoteTarball => directories::cached_tarball_folder_name_print( + &mut bufs.folder_name_buf, + self.url.slice(), + None, + ) + .as_bytes(), _ => unreachable!(), }; if folder_name.is_empty() || (folder_name.len() == 1 && folder_name[0] == b'/') { @@ -521,7 +571,7 @@ impl ExtractTarball { // Now that we've extracted the archive, we rename. #[cfg(windows)] - { + let moved: Result<(), Error> = 'moved: { // Windows EBUSY/SHARING_VIOLATION on `NtSetInformationFile` is // transient when a concurrent process (another `bun install` // sharing the cache, AV, the Search Indexer) is closing its @@ -537,6 +587,10 @@ impl ExtractTarball { } let path_to_use = path2; + let mut folder_name_z_buf = PathBuffer::uninit(); + folder_name_z_buf[0..folder_name.len()].copy_from_slice(folder_name); + folder_name_z_buf[folder_name.len()] = 0; + let folder_name_z = ZStr::from_buf(&folder_name_z_buf, folder_name.len()); loop { let dir_to_move = match sys::open_dir_at_windows_a( @@ -562,7 +616,7 @@ impl ExtractTarball { bun_fmt::s(folder_name), ), ); - return Err(crate::Error::InstallFailed); + break 'moved Err(crate::Error::InstallFailed); } }; @@ -582,6 +636,16 @@ impl ExtractTarball { // before we attempt to delete the destination, let's close the source dir. let _ = sys::close(dir_to_move); + if publish == CachePublish::KeepExisting + && sys::directory_exists_at( + cache_dir.fd(), + folder_name_z, + ) + .unwrap_or(false) + { + break; + } + // We tried to move the folder over // but it didn't work! // so instead of just simply deleting the folder @@ -595,12 +659,6 @@ impl ExtractTarball { .copy_from_slice(&[b't', b'm', b'p', 0]); let tempdest = ZStr::from_buf(&tempdest_buf, tmpname.len() + 3); - let mut folder_name_z_buf = PathBuffer::uninit(); - folder_name_z_buf[0..folder_name.len()] - .copy_from_slice(folder_name); - folder_name_z_buf[folder_name.len()] = 0; - let folder_name_z = - ZStr::from_buf(&folder_name_z_buf, folder_name.len()); match sys::renameat( Fd::from_std_dir(cache_dir), folder_name_z, @@ -637,7 +695,7 @@ impl ExtractTarball { bun_fmt::s(folder_name), ), ); - return Err(crate::Error::InstallFailed); + break 'moved Err(crate::Error::InstallFailed); } bun_sys::Result::Ok(_) => { let _ = sys::close(dir_to_move); @@ -646,33 +704,27 @@ impl ExtractTarball { break; } - } + Ok(()) + }; #[cfg(not(windows))] - { - // Attempt to gracefully handle duplicate concurrent `bun install` calls - // - // By: - // 1. Rename from temporary directory to cache directory and fail if it already exists - // 2a. If the rename fails, swap the cache directory with the temporary directory version - // 2b. Delete the temporary directory version ONLY if we're not using a provided temporary directory - // 3. If rename still fails, fallback to racily deleting the cache directory version and then renaming the temporary directory version again. - // - + let moved: Result<(), Error> = { if create_subdir { if let Some(folder) = bun_paths::Dirname::dirname(folder_name) { let _ = bun_sys::make_path::make_path(cache_dir, folder); } } - if let Err(err) = sys::renameat_concurrently_a( + sys::renameat_concurrently_a( tmpdir.fd(), tmpname.as_bytes(), cache_dir.fd(), folder_name, sys::RenameatConcurrentlyOptions { move_fallback: true, + keep_existing_destination: publish == CachePublish::KeepExisting, }, - ) { + ) + .map_err(|err| { log.add_error_fmt( None, bun_ast::Loc::EMPTY, @@ -684,9 +736,15 @@ impl ExtractTarball { bun_fmt::s(folder_name), ), ); - return Err(crate::Error::InstallFailed); - } + crate::Error::InstallFailed + }) + }; + + // `tmpname` now holds our own copy (failed or lost) or the swapped-out folder. + if moved.is_err() || publish != CachePublish::Replace { + let _ = tmpdir.delete_tree(tmpname.as_bytes()); } + moved?; // We return a resolved absolute absolute file path to the cache dir. // To get that directory, we open the directory again. diff --git a/src/install/hoisted_install.rs b/src/install/hoisted_install.rs index 81dc1dc5ee53..2b9c15432a7c 100644 --- a/src/install/hoisted_install.rs +++ b/src/install/hoisted_install.rs @@ -169,6 +169,14 @@ pub(crate) fn install_hoisted_packages( this.downloads_node = None; }); + // A partial install (security scanner) runs ahead of the real one and must not touch the layout. + let replacing_isolated_node_modules = packages_to_install.is_none() + && sys::directory_exists_at(Fd::cwd(), bun_paths::path_literal!("node_modules/.bun")) + .unwrap_or(false); + if replacing_isolated_node_modules { + crate::isolated_install::move_node_modules_aside(&this.lockfile); + } + // If there was already a valid lockfile and so we did not resolve, i.e. there was zero network activity // the packages could still not be in the cache dir // this would be a common scenario in a CI environment @@ -218,7 +226,8 @@ pub(crate) fn install_hoisted_packages( let mut skip_delete = new_node_modules; let mut skip_verify_installed_version_number = new_node_modules; - if this.options.enable.force_install() { + // Whatever `move_node_modules_aside` could not move is still deleted before it is reinstalled. + if this.options.enable.force_install() || replacing_isolated_node_modules { skip_verify_installed_version_number = true; skip_delete = false; } @@ -407,6 +416,8 @@ pub(crate) fn install_hoisted_packages( }), pending_installs: Vec::new(), install_count: 0, + replaced: Vec::new(), + inside_replaced_folder: false, }); } break 'trees trees.into_boxed_slice(); @@ -434,6 +445,7 @@ pub(crate) fn install_hoisted_packages( installer.node_modules.tree_id = node_modules.tree_id; let mut remaining: &[DependencyID] = node_modules.dependencies; installer.current_tree_id = node_modules.tree_id; + installer.set_inside_replaced_folder(node_modules.tree_id); // cache line is 64 bytes on ARM64 and x64 // PackageIDs are 4 bytes diff --git a/src/install/integrity.rs b/src/install/integrity.rs index e5ae223d335e..12e459448358 100644 --- a/src/install/integrity.rs +++ b/src/install/integrity.rs @@ -243,13 +243,11 @@ impl Integrity { impl fmt::Display for Integrity { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self.tag { - Tag::SHA1 => f.write_str("sha1-")?, - Tag::SHA256 => f.write_str("sha256-")?, - Tag::SHA384 => f.write_str("sha384-")?, - Tag::SHA512 => f.write_str("sha512-")?, - _ => return Ok(()), - } + let Some(algorithm) = self.tag.name() else { + return Ok(()); + }; + f.write_str(algorithm)?; + f.write_str("-")?; let mut base64_buf = [0u8; 512]; let bytes = self.slice(); @@ -293,6 +291,17 @@ impl Tag { self.0 >= Tag::SHA1.0 && self.0 <= Tag::SHA512.0 } + /// The algorithm part of the SRI string (`sha512` in `sha512-...`); `None` for `UNKNOWN`. + pub(crate) fn name(self) -> Option<&'static str> { + Some(match self { + Tag::SHA1 => "sha1", + Tag::SHA256 => "sha256", + Tag::SHA384 => "sha384", + Tag::SHA512 => "sha512", + _ => return None, + }) + } + pub(crate) fn parse(buf: &[u8]) -> (Tag, usize) { let Some(i) = strings::index_of_char(&buf[0..buf.len().min(7)], b'-') else { return (Tag::UNKNOWN, 0); diff --git a/src/install/isolated_install.rs b/src/install/isolated_install.rs index 98772c911a92..77b48472106e 100644 --- a/src/install/isolated_install.rs +++ b/src/install/isolated_install.rs @@ -144,8 +144,12 @@ impl<'a> run_tasks::RunTasksCallbacks for StoreRunTasksCallbacks<'a> { const HAS_ON_PACKAGE_DOWNLOAD_ERROR: bool = true; const IS_STORE_INSTALLER: bool = true; - fn on_extract_store_installer(ctx: &mut Self::Ctx, task_id: Task::Id) { - ctx.on_package_extracted(task_id); + fn on_extract_store_installer( + ctx: &mut Self::Ctx, + task_id: Task::Id, + data: &install::ExtractData, + ) { + ctx.on_package_extracted(task_id, data); } fn on_package_download_error_store( @@ -221,6 +225,85 @@ pub(crate) enum Timings { Quiet, } +/// `dep` is a `file:` dependency whose path is relative to the package declaring it +/// (resolved as `declarer`): `Package::from_npm` stores these paths as declared, +/// `Package::parse` and `overrides` store top-level relative ones. Such a folder is +/// linked from inside the package and is not a peer provider for the packages below. +fn dependency_is_contained_folder( + lockfile: &Lockfile, + declarer: &Resolution, + dep: &install::Dependency, +) -> bool { + if declarer.tag.is_local_package() || dep.version.tag != VersionTag::Folder { + return false; + } + let string_buf = lockfile.buffers.string_bytes.as_slice(); + !lockfile + .overrides + .contains_name(dep.name_hash, dep.name.slice(string_buf), string_buf) +} + +/// `dependency_is_contained_folder` for every `DependencyID`, and the `PackageID`s only such +/// dependencies resolve to: they get no store entry (`store::entry::Entry::nested_folder`). +fn contained_folders(lockfile: &Lockfile) -> Result<(DynamicBitSet, DynamicBitSet), AllocError> { + let pkgs = lockfile.packages.slice(); + let pkg_resolutions = pkgs.items_resolution(); + let resolutions = &lockfile.buffers.resolutions[..]; + let dependencies = &lockfile.buffers.dependencies[..]; + + let mut contained = DynamicBitSet::init_empty(dependencies.len())?; + let mut nested = DynamicBitSet::init_empty(pkg_resolutions.len())?; + for (pkg_id, pkg_res) in pkg_resolutions.iter().enumerate() { + if pkg_res.tag == ResolutionTag::Folder { + nested.set(pkg_id); + } + } + if nested.count() == 0 { + return Ok((contained, nested)); + } + + for (pkg_id, pkg_deps) in pkgs.items_dependencies().iter().enumerate() { + for dep_id in pkg_deps.begin()..pkg_deps.end() { + let target = resolutions[dep_id as usize]; + if target == invalid_package_id + || target as usize >= pkg_resolutions.len() + || pkg_resolutions[target as usize].tag != ResolutionTag::Folder + { + continue; + } + if dependency_is_contained_folder( + lockfile, + &pkg_resolutions[pkg_id], + &dependencies[dep_id as usize], + ) { + contained.set(dep_id as usize); + } else { + nested.unset(target as usize); + } + } + } + + Ok((contained, nested)) +} + +/// `pkg_id` has a `dependency_is_contained_folder` dependency resolving to `folder_pkg_id`. +pub(crate) fn folder_is_inside_package( + lockfile: &Lockfile, + pkg_id: PackageID, + folder_pkg_id: PackageID, +) -> bool { + let pkgs = lockfile.packages.slice(); + let declarer = &pkgs.items_resolution()[pkg_id as usize]; + let pkg_deps = pkgs.items_dependencies()[pkg_id as usize]; + let resolutions = &lockfile.buffers.resolutions[..]; + let dependencies = &lockfile.buffers.dependencies[..]; + + (pkg_deps.begin()..pkg_deps.end()).any(|dep_id| { + resolutions[dep_id as usize] == folder_pkg_id + && dependency_is_contained_folder(lockfile, declarer, &dependencies[dep_id as usize]) + }) +} + pub(crate) fn build_store( manager: &PackageManager, lockfile: &Lockfile, @@ -239,6 +322,9 @@ pub(crate) fn build_store( let dependencies = &lockfile.buffers.dependencies[..]; let string_buf = &lockfile.buffers.string_bytes[..]; + let (contained, nested_folders) = contained_folders(lockfile)?; + let is_contained_folder = |dep_id: DependencyID| contained.is_set(dep_id as usize); + let mut nodes: store::node::List = store::node::List::default(); // DFS so a deduplicated node's full subtree (and therefore its `peers`) @@ -313,15 +399,17 @@ pub(crate) fn build_store( }; if dep.behavior.is_peer() { own_peers.set(pkg_id as usize, bit); - } else if !is_filtered_dependency_or_workspace( - dep_id, - pkg_id, - workspace_filters, - install_root_dependencies, - manager, - lockfile, - resolutions, - ) { + } else if !is_contained_folder(dep_id) + && !is_filtered_dependency_or_workspace( + dep_id, + pkg_id, + workspace_filters, + install_root_dependencies, + manager, + lockfile, + resolutions, + ) + { provides.set(pkg_id as usize, bit); } } @@ -512,6 +600,7 @@ pub(crate) fn build_store( for ids in &node_dependencies[curr_id.get() as usize] { if dependencies[ids.dep_id as usize].name_hash == peer_name_hash + && !is_contained_folder(ids.dep_id) { break 'resolved ids.pkg_id; } @@ -589,7 +678,9 @@ pub(crate) fn build_store( let mut curr_id = entry.parent_id; 'walk: while curr_id != store::node::Id::INVALID { for ids in &node_dependencies[curr_id.get() as usize] { - if dependencies[ids.dep_id as usize].name_hash == peer_name_hash { + if dependencies[ids.dep_id as usize].name_hash == peer_name_hash + && !is_contained_folder(ids.dep_id) + { break 'walk; } } @@ -754,6 +845,11 @@ pub(crate) fn build_store( continue; } + // private to the ancestor; a package's own one still satisfies its own peer + if curr_id != node_id && is_contained_folder(ids.dep_id) { + continue; + } + let res = &pkg_resolutions[ids.pkg_id as usize]; if peer_dep.version.tag != VersionTag::Npm || res.tag != ResolutionTag::Npm @@ -1015,8 +1111,11 @@ pub(crate) fn build_store( let new_entry_parents: Vec = vec![entry.entry_parent_id]; + // never hoisted, so it must not claim the hoist slots for its name below + let new_entry_nested_folder = nested_folders.is_set(pkg_id as usize); + let hoisted = 'hoisted: { - if !manager.options.hoist { + if !manager.options.hoist || new_entry_nested_folder { break 'hoisted false; } @@ -1047,6 +1146,7 @@ pub(crate) fn build_store( parents: new_entry_parents, peer_hash: new_entry_peer_hash, hoisted, + nested_folder: new_entry_nested_folder, step: core::sync::atomic::AtomicU32::new(0), entry_hash: 0, scripts: core::cell::Cell::new(None), @@ -1088,7 +1188,7 @@ pub(crate) fn build_store( .name .slice(string_buf); public_hoisted.put(dep_name, ())?; - } else { + } else if !new_entry_nested_folder { // transitive dependencies (also direct dependencies of workspaces!) let dep_name = dependencies[new_entry_dep_id as usize] .name @@ -1118,6 +1218,10 @@ pub(crate) fn build_store( peers: node_peers[entry.node_id.get() as usize].clone(), }); + debug_assert!( + !new_entry_nested_folder || node_nodes[entry.node_id.get() as usize].is_empty(), + "a folder declared by a remote package is resolved without dependencies" + ); for &child_node_id in &node_nodes[entry.node_id.get() as usize] { entry_queue.write_item(QueuedEntry { node_id: child_node_id, @@ -1140,6 +1244,73 @@ pub(crate) fn build_store( }) } +/// Entry by entry rather than renaming `node_modules` itself, which may be a mount point or a symlink. +pub(crate) fn move_node_modules_aside(lockfile: &Lockfile) { + use bun_sys::FdExt as _; + + let mut old_modules = AutoRelPath::from(b"node_modules").assume_ok(); + let rand = fast_random(); + old_modules + .append_fmt(format_args!( + ".old_modules-{}", + bun_fmt::hex_lower(bun_core::bytes_of(&rand)) + )) + .assume_ok(); + if sys::mkdirat(Fd::cwd(), old_modules.slice_z(), 0o755).is_err() { + return; + } + let old_modules_name = old_modules.basename().to_vec(); + + let Ok(node_modules) = sys::open_dir_for_iteration(Fd::cwd(), b"node_modules") else { + return; + }; + let mut entry_path = AutoRelPath::from(b"node_modules").assume_ok(); + let mut iter = sys::iterate_dir(node_modules); + while let Ok(Some(entry)) = iter.next() { + let name = entry.name.slice_u8(); + if name == b".cache" || name == old_modules_name.as_slice() { + continue; + } + + let entry_path_len = entry_path.len(); + entry_path.append(name).assume_ok(); + let old_modules_len = old_modules.len(); + old_modules.append(name).assume_ok(); + + let _ = sys::renameat( + Fd::cwd(), + entry_path.slice_z(), + Fd::cwd(), + old_modules.slice_z(), + ); + + entry_path.set_length(entry_path_len); + old_modules.set_length(old_modules_len); + } + node_modules.close(); + + for workspace_path in lockfile.workspace_paths.values() { + let mut workspace_node_modules = + AutoRelPath::from(workspace_path.slice(&lockfile.buffers.string_bytes)).assume_ok(); + let basename = workspace_node_modules.basename().to_vec(); + workspace_node_modules.append(b"node_modules").assume_ok(); + + let old_modules_len = old_modules.len(); + old_modules + .append_fmt(format_args!(".old_{}_modules", BStr::new(&basename))) + .assume_ok(); + + let _ = sys::renameat( + Fd::cwd(), + workspace_node_modules.slice_z(), + Fd::cwd(), + old_modules.slice_z(), + ); + + old_modules.set_length(old_modules_len); + } +} + /// Runs on main thread pub(crate) fn install_isolated_packages( manager: &mut PackageManager, @@ -1690,233 +1861,13 @@ pub(crate) fn install_isolated_packages( // matches `Installer::NODE_MODULES_BUN`. let bun_modules_path = paths::path_literal!("node_modules/.bun"); - match sys::mkdirat(Fd::cwd(), node_modules_path, 0o755) { - Ok(()) => { - // fallthrough to creating bun_modules below + if sys::mkdirat(Fd::cwd(), node_modules_path, 0o755).is_err() { + if sys::directory_exists_at(Fd::cwd(), bun_modules_path).unwrap_or(false) { + break 'is_new_bun_modules false; } - Err(_) => { - match sys::mkdirat(Fd::cwd(), bun_modules_path, 0o755) { - Err(_) => break 'is_new_bun_modules false, - Ok(()) => {} - } - - // 'node_modules' exists and 'node_modules/.bun' doesn't - - #[cfg(windows)] - { - // Windows: - // 1. create 'node_modules/.old_modules-{hex}' - // 2. for each entry in 'node_modules' rename into 'node_modules/.old_modules-{hex}' - // 3. for each workspace 'node_modules' rename into 'node_modules/.old_modules-{hex}/old_{basename}_modules' - - // `sys::mkdirat`/`renameat` take `&ZStr` (u8) and widen - // internally, so a single u8 `AutoRelPath` covers both the - // mkdir and rename targets. - let mut rename_path = AutoRelPath::from(b"node_modules").assume_ok(); - let rand = fast_random(); - rename_path - .append_fmt(format_args!( - ".old_modules-{}", - bun_fmt::hex_lower(bun_core::bytes_of(&rand)) - )) - .assume_ok(); - - // 1 - if sys::mkdirat(Fd::cwd(), rename_path.slice_z(), 0o755).is_err() { - break 'is_new_bun_modules true; - } - let Ok(node_modules) = sys::open_dir_for_iteration(Fd::cwd(), b"node_modules") - else { - break 'is_new_bun_modules true; - }; - // Windows HANDLE-leak audit: `Fd` is `Copy` (no Drop) and the - // `WrappedIterator` from `sys::iterate_dir` does not own/close it, - // so close explicitly. The guard fires on - // normal fall-through to step 3 and on every - // `break 'is_new_bun_modules true` early exit. - let _close_node_modules = scopeguard::guard(node_modules, |fd| { - use bun_sys::FdExt as _; - fd.close(); - }); - - let mut entry_path = AutoRelPath::from(b"node_modules").assume_ok(); - - // 2 - let mut node_modules_iter = sys::iterate_dir(node_modules); - loop { - let Some(entry) = (match node_modules_iter.next() { - Ok(v) => v, - Err(_) => break 'is_new_bun_modules true, - }) else { - break; - }; - if bun_core::starts_with_char(entry.name.slice_u8(), b'.') { - continue; - } - - // Capture lengths and truncate manually so - // the paths stay unborrowed across the loop body. - let entry_path_save = entry_path.len(); - entry_path.append(entry.name.slice()).assume_ok(); - - let rename_path_save = rename_path.len(); - rename_path.append(entry.name.slice()).assume_ok(); - - let _ = sys::renameat( - Fd::cwd(), - entry_path.slice_z(), - Fd::cwd(), - rename_path.slice_z(), - ); - - rename_path.set_length(rename_path_save); - entry_path.set_length(entry_path_save); - } - - // 3 - for workspace_path in lockfile.workspace_paths.values() { - let mut workspace_node_modules = - AutoRelPath::from(workspace_path.slice(&lockfile.buffers.string_bytes)) - .assume_ok(); - - // Clone basename before mutating - // `workspace_node_modules`. - let basename = workspace_node_modules.basename().to_vec(); - - workspace_node_modules.append(b"node_modules").assume_ok(); - - // Reshaped for borrowck — capture length instead - // of `save()` so `rename_path` stays unborrowed. - let rename_path_save = rename_path.len(); - rename_path - .append_fmt(format_args!(".old_{}_modules", BStr::new(&basename))) - .assume_ok(); - - let _ = sys::renameat( - Fd::cwd(), - workspace_node_modules.slice_z(), - Fd::cwd(), - rename_path.slice_z(), - ); - - rename_path.set_length(rename_path_save); - } - } - #[cfg(not(windows))] - { - // Posix: - // 1. rename existing 'node_modules' to temp location - // 2. create new 'node_modules' directory - // 3. rename temp into 'node_modules/.old_modules-{hex}' - // 4. attempt renaming 'node_modules/.old_modules-{hex}/.cache' to 'node_modules/.cache' - // 5. rename each workspace 'node_modules' into 'node_modules/.old_modules-{hex}/old_{basename}_modules' - let mut temp_node_modules_buf = PathBuffer::uninit(); - let temp_node_modules = paths::fs::FileSystem::tmpname( - b"tmp_modules", - &mut temp_node_modules_buf.0, - fast_random(), - ) - .expect("unreachable"); - - // 1 - if sys::renameat( - Fd::cwd(), - bun_core::zstr!("node_modules"), - Fd::cwd(), - temp_node_modules, - ) - .is_err() - { - break 'is_new_bun_modules true; - } - - // 2 - if let Err(err) = sys::mkdirat(Fd::cwd(), node_modules_path, 0o755) { - Output::err(err, "failed to create './node_modules'", format_args!("")); - Global::exit(1); - } - - if let Err(err) = sys::mkdirat(Fd::cwd(), bun_modules_path, 0o755) { - Output::err( - err, - "failed to create './node_modules/.bun'", - format_args!(""), - ); - Global::exit(1); - } - - let mut rename_path = AutoRelPath::from(b"node_modules").assume_ok(); - - let rand = fast_random(); - rename_path - .append_fmt(format_args!( - ".old_modules-{}", - bun_fmt::hex_lower(bun_core::bytes_of(&rand)) - )) - .assume_ok(); - - // 3 - if sys::renameat( - Fd::cwd(), - temp_node_modules, - Fd::cwd(), - rename_path.slice_z(), - ) - .is_err() - { - break 'is_new_bun_modules true; - } - - rename_path.append(b".cache").assume_ok(); - - let mut cache_path = AutoRelPath::from(b"node_modules").assume_ok(); - cache_path.append(b".cache").assume_ok(); - - // 4 - let _ = sys::renameat( - Fd::cwd(), - rename_path.slice_z(), - Fd::cwd(), - cache_path.slice_z(), - ); - - // remove .cache so we can append destination for each workspace - rename_path.undo(1); - - // 5 - for workspace_path in lockfile.workspace_paths.values() { - let mut workspace_node_modules = - AutoRelPath::from(workspace_path.slice(&lockfile.buffers.string_bytes)) - .assume_ok(); - - // Clone basename before mutating - // `workspace_node_modules`. - let basename = workspace_node_modules.basename().to_vec(); - - workspace_node_modules.append(b"node_modules").assume_ok(); - - // Capture the length and truncate manually - // so `rename_path` stays unborrowed between save/restore. - let rename_path_save = rename_path.len(); - - rename_path - .append_fmt(format_args!(".old_{}_modules", BStr::new(&basename))) - .assume_ok(); - - let _ = sys::renameat( - Fd::cwd(), - workspace_node_modules.slice_z(), - Fd::cwd(), - rename_path.slice_z(), - ); - - rename_path.set_length(rename_path_save); - } - } - - break 'is_new_bun_modules true; - } + // 'node_modules' exists and 'node_modules/.bun' doesn't + move_node_modules_aside(lockfile); } if let Err(err) = sys::mkdirat(Fd::cwd(), bun_modules_path, 0o755) { @@ -1983,6 +1934,7 @@ pub(crate) fn install_isolated_packages( let entry_steps = entries.items_step(); let entry_dependencies = entries.items_dependencies(); let entry_hoisted = entries.items_hoisted(); + let entry_nested_folder = entries.items_nested_folder(); // Reborrow through a // `BackRef` so `string_buf` / `pkgs` don't tie up `&mut lockfile` for @@ -2026,6 +1978,7 @@ pub(crate) fn install_isolated_packages( installer: bun_ptr::BackRef::from(core::ptr::NonNull::dangling()), result: installer::Result::None, relink: installer::Relink::Off, + blocked_scripts: 0, task: bun_threading::thread_pool::Task { callback: installer::Task::callback, node: Default::default(), @@ -2074,6 +2027,7 @@ pub(crate) fn install_isolated_packages( global_store_tmp_suffix: fast_random(), summary: Default::default(), task_queue: Default::default(), + failed_optional_entries: Vec::new(), }; // No long-lived `&mut PackageManager` reborrow here — `installer.start_task()`, // `on_task_complete()`, and `on_task_fail()` below all reach the manager through @@ -2095,7 +2049,7 @@ pub(crate) fn install_isolated_packages( task.installer = installer_backref; } - // `append_store_path` runs on worker threads via `&Installer` and + // `append_dependency_path` runs on worker threads via `&Installer` and // can't take `&mut PackageManager` there, so ensure the // global link dir once on the main thread before any `.symlink` // resolution can be reached by a task. Guarded so installs without @@ -2146,7 +2100,7 @@ pub(crate) fn install_isolated_packages( if let Some(name) = unsafe_folder_name { Output::err_generic( "\"{}\" is not a valid install folder name", - (BStr::new(name),), + (bun_core::fmt::escape_control_chars(name),), ); Output::flush(); Global::exit(1); @@ -2193,10 +2147,56 @@ pub(crate) fn install_isolated_packages( // .monotonic is okay because the task isn't running on another thread. entry_steps[entry_id.get() as usize] .store(installer::Step::Done as u32, Ordering::Relaxed); - installer.on_task_complete(entry_id, installer::CompleteState::Skipped); + + // Same target check as the hoisted linker's `install_from_link`. + let mut link_target = bun_paths::AutoAbsPath::init_top_level_dir(); + let checked = installer + .append_dependency_path(&mut link_target, entry_id) + .and_then(|()| { + sys::openat( + Fd::cwd(), + link_target.slice_z(), + sys::O::RDONLY | sys::O::DIRECTORY, + 0, + ) + .map_err(installer::TaskError::LinkPackage) + }); + match checked { + Ok(fd) => { + use bun_sys::FdExt as _; + fd.close(); + installer.on_task_complete(entry_id, installer::CompleteState::Skipped); + } + Err(err) => installer.on_task_fail(entry_id, &err), + } continue; } ResolutionTag::Folder => { + if entry_nested_folder[entry_id.get() as usize] { + // linked by the packages containing it (`Installer::symlink_dependencies`) + debug_assert!(entry_dependencies[entry_id.get() as usize].list.is_empty()); + let folder = pkg_res.folder().slice(string_buf); + let state = if crate::bin::bin_target_escapes_package_dir(folder) { + // the resolver rejects these; only an edited or old lockfile gets here + Output::err_generic( + "refusing to install dependency {} with unsafe folder path \"{}\"", + (BStr::new(pkg_name.slice(string_buf)), BStr::new(folder)), + ); + Output::flush(); + if installer.manager().options.enable.fail_early() { + Global::exit(1); + } + installer::CompleteState::Fail + } else { + installer::CompleteState::Skipped + }; + // .monotonic is okay because the task isn't running on another thread. + entry_steps[entry_id.get() as usize] + .store(installer::Step::Done as u32, Ordering::Relaxed); + installer.on_task_complete(entry_id, state); + continue; + } + // folders are always hardlinked to keep them up-to-date installer.start_task(entry_id); continue; @@ -2236,6 +2236,7 @@ pub(crate) fn install_isolated_packages( .ok() .unwrap_or(false); } + // Likewise, the package directory appears here only once fully linked. installer.append_real_store_path(&mut store_path, entry_id, installer::Which::Final); // Capture the length instead of a `ResetScope` so // `store_path` stays unborrowed. @@ -2349,11 +2350,12 @@ pub(crate) fn install_isolated_packages( pkg_res.github(), None, ), - ResolutionTag::LocalTarball => package_manager::cached_tarball_folder_name( - installer.manager(), - *pkg_res.local_tarball(), - None, - ), + ResolutionTag::LocalTarball => { + package_manager::cached_local_tarball_folder_name( + &pkgs.items_meta()[pkg_id as usize].integrity, + None, + ) + } ResolutionTag::RemoteTarball => { package_manager::cached_tarball_folder_name( installer.manager(), @@ -2370,6 +2372,8 @@ pub(crate) fn install_isolated_packages( let missing_from_cache = match installer.manager().get_preinstall_state(pkg_id) { + // no entry name until extraction records a local tarball's integrity + _ if cache_subpath_z.is_empty() => true, install::PreinstallState::Done => false, _ => { let exists = package_manager::directories::is_package_in_cache_at( @@ -2441,7 +2445,9 @@ pub(crate) fn install_isolated_packages( "failed to enqueue package for download: {}@{}", ( BStr::new(pkg_name.slice(string_buf)), - pkg_res.fmt(string_buf, bun_fmt::PathSep::Auto), + bun_fmt::EscapeControlChars( + pkg_res.fmt(string_buf, bun_fmt::PathSep::Auto), + ), ), ); Output::flush(); @@ -2499,7 +2505,9 @@ pub(crate) fn install_isolated_packages( "failed to enqueue github package for download: {}@{}", ( BStr::new(pkg_name.slice(string_buf)), - pkg_res.fmt(string_buf, bun_fmt::PathSep::Auto), + bun_fmt::EscapeControlChars( + pkg_res.fmt(string_buf, bun_fmt::PathSep::Auto), + ), ), ); Output::flush(); @@ -2552,7 +2560,9 @@ pub(crate) fn install_isolated_packages( "failed to enqueue tarball for download: {}@{}", ( BStr::new(pkg_name.slice(string_buf)), - pkg_res.fmt(string_buf, bun_fmt::PathSep::Auto), + bun_core::fmt::for_terminal( + pkg_res.fmt(string_buf, bun_fmt::PathSep::Auto), + ), ), ); Output::flush(); @@ -2602,6 +2612,8 @@ pub(crate) fn install_isolated_packages( } } + installer.unlink_failed_optional_entries(); + if installer.manager().options.log_level.show_progress() { progress.root.end(); *progress = Progress::default(); diff --git a/src/install/isolated_install/Installer.rs b/src/install/isolated_install/Installer.rs index ce346a116b37..a7af56e5c4f8 100644 --- a/src/install/isolated_install/Installer.rs +++ b/src/install/isolated_install/Installer.rs @@ -3,6 +3,7 @@ use std::io::Write as _; use bun_ast::Log; use bun_collections::{ArrayHashMap, DynamicBitSet, StringHashMap}; +use bun_core::fmt::EscapeControlChars; use bun_core::{Environment, Global, Output}; use bun_core::{ZStr, strings}; use bun_paths::{self as paths, AbsPath, AutoAbsPath, AutoRelPath}; @@ -15,7 +16,9 @@ use bun_sys::{FdDirExt as _, FdExt as _}; use crate::bin_real; use crate::lockfile::package; use crate::lockfile_real::PackageIDSlice; -use crate::package_install::{Method as InstallMethod, Summary as InstallSummary}; +use crate::package_install::{ + Method as InstallMethod, StagingPath, Summary as InstallSummary, rename_staging_into_place, +}; use crate::package_manager_real::Command; use crate::postinstall_optimizer; use crate::postinstall_optimizer::PostinstallOptimizer; @@ -36,7 +39,7 @@ use super::symlinker::{self, Symlinker}; use crate::bun_fs; use crate::lockfile_real::package::PackageColumns as _; use crate::package_manager_real::directories; -use crate::package_manager_real::package_manager_options::Do; +use crate::package_manager_real::package_manager_options::{Do, Enable}; /// The enum lives at module level in `crate::resolution`. type ResolutionTag = resolution::Tag; @@ -70,7 +73,8 @@ pub struct Installer<'a> { /// pool and each task derefs this field; a `&'a mut` would assert /// exclusivity every concurrent task violates. Mutated only for /// `lockfile.trusted_dependencies` (under `trusted_dependencies_mutex`, - /// narrowed via `addr_of_mut!`). Never null. Read via `lockfile()`. + /// narrowed via `addr_of_mut!`) and in `record_extracted_integrity` (main + /// thread, one row). Never null. Read via `lockfile()`. pub lockfile: *mut Lockfile, pub(crate) summary: InstallSummary, @@ -117,6 +121,9 @@ pub struct Installer<'a> { /// Main-thread only: `waiters_head[dep]` starts the intrusive list of blocked entries waiting on `dep`, linked through `next_waiter`. pub(crate) waiters_head: Box<[StoreEntryId]>, pub(crate) next_waiter: Box<[StoreEntryId]>, + + /// Main-thread only: entries deleted by `on_optional_dependency_scripts_failed`. + pub(crate) failed_optional_entries: Vec, } impl<'a> Installer<'a> { @@ -181,7 +188,11 @@ impl<'a> Installer<'a> { self.start_task(entry_id); } - pub(crate) fn on_package_extracted(&mut self, task_id: crate::package_manager_task::Id) { + pub(crate) fn on_package_extracted( + &mut self, + task_id: crate::package_manager_task::Id, + data: &install::ExtractData, + ) { if let Some(removed) = self.manager_mut().task_queue.remove(&task_id) { let store = self.store; @@ -205,6 +216,11 @@ impl<'a> Installer<'a> { let node_id = entry_node_ids[entry_id.get() as usize]; let pkg_id = node_pkg_ids[node_id.get() as usize]; + + if data.integrity.tag.is_supported() { + self.record_extracted_integrity(pkg_id, &data.integrity); + } + let pkg_name = pkg_names[pkg_id as usize]; let pkg_name_hash = pkg_name_hashes[pkg_id as usize]; let pkg_res = &pkg_resolutions[pkg_id as usize]; @@ -230,6 +246,34 @@ impl<'a> Installer<'a> { } } + /// Main thread, before the package's tasks start (they name a local tarball's cache + /// entry after it). Same write-back as `install_enqueued_packages_after_extraction`. + fn record_extracted_integrity(&mut self, pkg_id: PackageID, integrity: &install::Integrity) { + let pkgs = self.lockfile().packages.slice(); + assert!((pkg_id as usize) < pkgs.len()); + // SAFETY: in bounds; raw column pointer, so no `&mut Lockfile` is formed. Only this + // package's tasks read its `integrity` and none has started; other tasks' `&[Meta]` + // borrows never touch these bytes. + let recorded = unsafe { + core::ptr::addr_of_mut!( + (*pkgs + .items_raw::<"meta", package::Meta>() + .add(pkg_id as usize)) + .integrity + ) + }; + // SAFETY: see above. + if unsafe { (*recorded).tag.is_supported() } { + return; + } + // SAFETY: see above. + unsafe { recorded.write(*integrity) }; + self.manager_mut() + .options + .enable + .set(Enable::FORCE_SAVE_LOCKFILE, true); + } + /// Called from main thread when a tarball download or extraction fails. /// Without this, the upfront pending-task slot for each waiting entry is /// never released and the install loop blocks forever on @@ -268,10 +312,12 @@ impl<'a> Installer<'a> { Output::err_generic( "failed to download {}@{}: {}\n {}", ( - bstr::BStr::new(name), - resolution.fmt(string_buf, bun_core::fmt::PathSep::Auto), + bun_core::fmt::escape_control_chars(name), + bun_core::fmt::for_terminal( + resolution.fmt(string_buf, bun_core::fmt::PathSep::Auto), + ), bstr::BStr::new(download_error_reason(err)), - bstr::BStr::new(url), + EscapeControlChars(bun_core::fmt::redacted_npm_url(url)), ), ); Output::flush(); @@ -331,58 +377,70 @@ impl<'a> Installer<'a> { let node_id = entry_node_ids[entry_id.get() as usize]; let pkg_id = node_pkg_ids[node_id.get() as usize]; - let pkg_name = pkg_names[pkg_id as usize]; let pkg_res = pkg_resolutions[pkg_id as usize]; + let pkg_name = + bun_core::fmt::escape_control_chars(pkg_names[pkg_id as usize].slice(string_buf)); + let pkg_res_fmt = + bun_core::fmt::for_terminal(pkg_res.fmt(string_buf, bun_core::fmt::PathSep::Auto)); match err { TaskError::LinkPackage(link_err) => { Output::err( link_err.clone(), "failed to link package: {}@{}", - ( - bstr::BStr::new(pkg_name.slice(string_buf)), - pkg_res.fmt(string_buf, bun_core::fmt::PathSep::Auto), - ), + (&pkg_name, &pkg_res_fmt), ); } TaskError::SymlinkDependencies(symlink_err) => { Output::err( symlink_err.clone(), "failed to symlink dependencies for package: {}@{}", - ( - bstr::BStr::new(pkg_name.slice(string_buf)), - pkg_res.fmt(string_buf, bun_core::fmt::PathSep::Auto), - ), + (&pkg_name, &pkg_res_fmt), ); } - TaskError::Patching(patch_log) => { - Output::err_generic( - "failed to patch package: {}@{}", - ( - bstr::BStr::new(pkg_name.slice(string_buf)), - pkg_res.fmt(string_buf, bun_core::fmt::PathSep::Auto), - ), + TaskError::LinkPathTooLong => { + Output::err( + "ENAMETOOLONG", + "link path for package {} is too long", + (&pkg_name,), ); + } + TaskError::Patching(patch_log) => { + Output::err_generic("failed to patch package: {}@{}", (&pkg_name, &pkg_res_fmt)); let _ = patch_log.print(std::ptr::from_mut(Output::error_writer())); } TaskError::Binaries(bin_err) => { Output::err( *bin_err, "failed to link binaries for package: {}@{}", - ( - bstr::BStr::new(pkg_name.slice(string_buf)), - pkg_res.fmt(string_buf, bun_core::fmt::PathSep::Auto), - ), + (&pkg_name, &pkg_res_fmt), ); } + TaskError::DependencyBinaries(dep_errs) => { + for &(dep_entry_id, dep_err) in dep_errs.iter() { + let dep_node_id = entry_node_ids[dep_entry_id.get() as usize]; + let dep_pkg_id = node_pkg_ids[dep_node_id.get() as usize]; + Output::err( + dep_err, + "failed to link binaries of dependency {}@{} for package: {}@{}", + ( + bstr::BStr::new(pkg_names[dep_pkg_id as usize].slice(string_buf)), + pkg_resolutions[dep_pkg_id as usize] + .fmt(string_buf, bun_core::fmt::PathSep::Auto), + &pkg_name, + &pkg_res_fmt, + ), + ); + } + } TaskError::Download(dl) => { Output::err_generic( "failed to download {}@{}: {}\n {}", ( - bstr::BStr::new(pkg_name.slice(string_buf)), - pkg_res.fmt(string_buf, bun_core::fmt::PathSep::Auto), + &pkg_name, + &pkg_res_fmt, bstr::BStr::new(download_error_reason(dl.err)), - bstr::BStr::new(&dl.url), + EscapeControlChars(bun_core::fmt::redacted_npm_url(&dl.url)), ), ); } @@ -397,35 +455,18 @@ impl<'a> Installer<'a> { let mut staging = AutoAbsPath::init(); self.append_global_store_entry_path(&mut staging, entry_id, Which::Staging); let _ = Fd::cwd().delete_tree(staging.slice()); - } - - // attempt deleting the package so the next install will install it again - match pkg_res.tag { - ResolutionTag::Uninitialized - | ResolutionTag::SingleFileModule - | ResolutionTag::Root - | ResolutionTag::Workspace - | ResolutionTag::Symlink => {} - - // to be safe make sure we only delete packages in the store + } else if matches!( + pkg_res.tag, ResolutionTag::Npm - | ResolutionTag::Git - | ResolutionTag::Github - | ResolutionTag::LocalTarball - | ResolutionTag::RemoteTarball - | ResolutionTag::Folder => { - let mut store_path = AutoRelPath::init(); - - // OOM/capacity: fire-and-forget - let _ = store_path.append_fmt(format_args!( - "node_modules/{}", - store::entry::fmt_store_path(entry_id, self.store, self.lockfile()), - )); - - let _ = sys::unlink(store_path.slice_z()); - } - - _ => {} + | ResolutionTag::Git + | ResolutionTag::Github + | ResolutionTag::LocalTarball + | ResolutionTag::RemoteTarball + ) { + // A failed link only ever wrote to the staging directory. + let mut staging = AutoPath::init_top_level_dir(); + self.append_real_store_path(&mut staging, entry_id, Which::Staging); + let _ = Fd::cwd().delete_tree(staging.slice()); } if self.manager().options.enable.fail_early() { @@ -565,6 +606,26 @@ impl<'a> Installer<'a> { let is_duplicate = self.installed.is_set(pkg_id as usize); self.summary.success += (!is_duplicate) as u32; self.installed.set(pkg_id as usize); + + // Once per package, like `success`: a package has one store entry per peer set. + let blocked_scripts = self.tasks[entry_id.get() as usize].blocked_scripts; + if blocked_scripts > 0 && !is_duplicate { + let dep_id = nodes.items_dep_id()[node_id.get() as usize]; + let dep_name_hash = self.lockfile().buffers.dependencies[dep_id as usize].name_hash; + *self + .summary + .packages_with_blocked_scripts + .entry(dep_name_hash as TruncatedPackageNameHash) + .or_default() += usize::from(blocked_scripts); + } + } + + /// Called from main thread once the package is deleted; dependents finish without it. + pub(crate) fn on_optional_dependency_scripts_failed(&mut self, entry_id: StoreEntryId) { + self.failed_optional_entries.push(entry_id); + self.store.entries.items_step()[entry_id.get() as usize] + .store(Step::Done as u32, Ordering::Release); + self.on_task_complete(entry_id, CompleteState::Skipped); } /// Main thread only: `completed` just reached `Step::Done`; re-check every entry waiting on it. @@ -656,6 +717,8 @@ pub struct Task { pub(crate) result: Result, pub(crate) relink: Relink, + /// Set by `Step::RunPreinstall` on the task thread, read by `on_task_complete` like `relink`. + pub(crate) blocked_scripts: u8, } // SAFETY: `next` is the sole intrusive link for `UnboundedQueue`. @@ -687,8 +750,12 @@ pub struct DownloadError { pub enum TaskError { LinkPackage(sys::Error), SymlinkDependencies(sys::Error), + /// See `Installer::append_dependency_path`. + LinkPathTooLong, RunScripts(crate::Error), Binaries(crate::Error), + /// (dependency entry, error) + DependencyBinaries(Box<[(StoreEntryId, crate::Error)]>), Patching(Log), Download(DownloadError), } @@ -698,7 +765,9 @@ impl TaskError { match self { TaskError::LinkPackage(err) => TaskError::LinkPackage(err.clone()), TaskError::SymlinkDependencies(err) => TaskError::SymlinkDependencies(err.clone()), + TaskError::LinkPathTooLong => TaskError::LinkPathTooLong, TaskError::Binaries(err) => TaskError::Binaries(*err), + TaskError::DependencyBinaries(errs) => TaskError::DependencyBinaries(errs.clone()), TaskError::RunScripts(err) => TaskError::RunScripts(*err), TaskError::Patching(_log) => { // `bun_ast::Log` is non-Clone; the only caller of @@ -1132,9 +1201,12 @@ impl Task { patch_info.contents_hash(), ), ResolutionTag::LocalTarball => { - directories::cached_tarball_folder_name( - manager, - *pkg_res.local_tarball(), + // recorded by the lockfile or `on_package_extracted` + debug_assert!( + pkg_metas[pkg_id as usize].integrity.tag.is_supported() + ); + directories::cached_local_tarball_folder_name( + &pkg_metas[pkg_id as usize].integrity, patch_info.contents_hash(), ) } @@ -1210,24 +1282,7 @@ impl Task { } }; if is_stale_link { - let remove_err: Option = { - #[cfg(windows)] - { - 'win: { - if let Some(_e) = sys::rmdir(local.slice_z()).err() { - if let Some(e) = sys::unlink(local.slice_z()).err() { - break 'win Some(e); - } - } - break 'win None; - } - } - #[cfg(not(windows))] - { - sys::unlink(local.slice_z()).err() - } - }; - if let Some(e) = remove_err { + if let Err(e) = remove_link(local.slice_z()) { if e.get_errno() != sys::Errno::ENOENT { // Do NOT proceed: the backend below would // write *through* the still-live symlink @@ -1237,14 +1292,15 @@ impl Task { } } - // hardlink/copyfile overlay an existing tree, keeping files a previous patched build added. - let mut previous = AutoPath::init_top_level_dir(); - installer.append_real_store_path( - &mut previous, - self.entry_id, - Which::Final, - ); - let _ = Fd::cwd().delete_tree(previous.slice()); + // The final path must be free for the rename below, and the backends + // would keep whatever a stale staging tree holds. + for which in [Which::Final, Which::Staging] { + let mut leftover = AutoPath::init_top_level_dir(); + installer.append_real_store_path(&mut leftover, self.entry_id, which); + if let sys::Result::Err(err) = Fd::cwd().delete_tree(leftover.slice()) { + return Ok(Yield::failure(TaskError::LinkPackage(err))); + } + } } if uses_global_store { @@ -1349,8 +1405,7 @@ impl Task { }, } - step = self.next_step(current_step); - continue 'step; + break 'backend; } } @@ -1419,8 +1474,7 @@ impl Task { } } - step = self.next_step(current_step); - continue 'step; + break 'backend; } // fallthrough copyfile @@ -1487,12 +1541,21 @@ impl Task { } } - step = self.next_step(current_step); - continue 'step; + break 'backend; } } } - // unreachable: every backend arm continues to next_step or returns + + if !uses_global_store { + if let sys::Result::Err(err) = + installer.commit_local_store_package(self.entry_id) + { + return Ok(Yield::failure(TaskError::LinkPackage(err))); + } + } + + step = self.next_step(current_step); + continue 'step; } Step::SymlinkDependencies => { @@ -1507,10 +1570,8 @@ impl Task { }; let changed = match installer.symlink_dependencies(self.entry_id, strategy) { - sys::Result::Ok(changed) => changed, - sys::Result::Err(err) => { - return Ok(Yield::failure(TaskError::SymlinkDependencies(err))); - } + Ok(changed) => changed, + Err(err) => return Ok(Yield::failure(err)), }; if relinking { @@ -1555,7 +1616,7 @@ impl Task { Step::SymlinkDependencyBinaries => { let current_step = Step::SymlinkDependencyBinaries; if let Err(err) = installer.link_dependency_bins(self.entry_id) { - return Ok(Yield::failure(TaskError::Binaries(err))); + return Ok(Yield::failure(err)); } match pkg_res.tag { @@ -1587,22 +1648,7 @@ impl Task { Step::RunPreinstall => { let current_step = Step::RunPreinstall; - if !manager_ref.options.do_.contains(Do::RUN_SCRIPTS) - || self.entry_id == StoreEntryId::ROOT - { - step = self.next_step(current_step); - continue; - } - - // The eligibility check excludes any package whose - // lifecycle scripts are trusted to run, so a global-store - // entry should never reach script enqueueing. Guard it - // anyway: `meta.hasInstallScript` can be a false negative - // (yarn-migrated lockfiles force it to `.false`), and a - // script running with cwd inside a shared content- - // addressed directory would mutate every other project's - // copy. - if installer.entry_uses_global_store(self.entry_id) { + if self.entry_id == StoreEntryId::ROOT { step = self.next_step(current_step); continue; } @@ -1630,15 +1676,36 @@ impl Task { break 'brk (false, false); }; + if !(pkg_res.tag != ResolutionTag::Root + && (pkg_res.tag == ResolutionTag::Workspace || is_trusted)) + { + // Before the early returns below, like the hoisted installer. + if pkg_res.tag != ResolutionTag::Root + && pkg_metas[pkg_id as usize].has_install_script() + { + self.blocked_scripts = self.count_blocked_scripts( + installer, + pkg_script_lists[pkg_id as usize], + dep.name.slice(string_buf), + &pkg_res, + ); + } + step = self.next_step(current_step); + continue; + } + + // Trusted entries are never global-store eligible; never run in the shared dir. + if !manager_ref.options.do_.contains(Do::RUN_SCRIPTS) + || installer.entry_uses_global_store(self.entry_id) + { + step = self.next_step(current_step); + continue; + } + let mut pkg_cwd = AutoAbsPath::init_top_level_dir(); installer.append_store_path(&mut pkg_cwd, self.entry_id); 'enqueue_lifecycle_scripts: { - if !(pkg_res.tag != ResolutionTag::Root - && (pkg_res.tag == ResolutionTag::Workspace || is_trusted)) - { - break 'enqueue_lifecycle_scripts; - } let mut pkg_scripts: package::scripts::Scripts = pkg_script_lists[pkg_id as usize]; let manager = manager_ref.get(); @@ -1660,6 +1727,7 @@ impl Task { pkg_metas, manager.options.cpu, manager.options.os, + manager.options.libc, ) { break 'enqueue_lifecycle_scripts; @@ -1853,6 +1921,8 @@ impl Task { bin_linker.target_node_modules_path = bin_linker.node_modules_path; bin_linker.target_package_name = strings::StringOrTinyString::init(dep_name); + bin_linker.err = None; + bin_linker.skipped_due_to_missing_bin = false; if manager_ref.options.log_level.is_verbose() { bun_core::pretty_errorln!( @@ -1937,6 +2007,54 @@ impl Task { } } + /// Task thread. Counts the installed scripts; `hasInstallScript` alone can be a false positive. + fn count_blocked_scripts( + &self, + installer: &Installer<'_>, + mut pkg_scripts: package::scripts::Scripts, + dep_name: &[u8], + pkg_res: &Resolution, + ) -> u8 { + let (manager, lockfile) = (installer.manager(), installer.lockfile()); + // A global-store entry stays in its staging dir until `Step::Binaries` renames it; a + // project-local one was renamed into place by `Step::LinkPackage`. + let mut pkg_dir = AutoAbsPath::init_top_level_dir(); + let which = if installer.entry_uses_global_store(self.entry_id) { + Which::Staging + } else { + Which::Final + }; + installer.append_real_store_path(&mut pkg_dir, self.entry_id, which); + + let mut log = Log::init(); + match pkg_scripts.get_list(&mut log, lockfile, &mut pkg_dir, dep_name, pkg_res) { + Ok(Some(list)) => { + if manager.options.log_level.is_verbose() { + bun_core::pretty_error!( + "Blocked {} scripts for: {}@{}\n", + list.total, + bstr::BStr::new(dep_name), + pkg_res.fmt( + lockfile.buffers.string_bytes.as_slice(), + bun_core::fmt::PathSep::Posix + ), + ); + } + list.total + } + Ok(None) => 0, + Err(err) => { + if !manager.options.log_level.is_silent() { + Output::err_generic( + "failed to fill lifecycle scripts for {}: {}", + (bstr::BStr::new(dep_name), err.name()), + ); + } + 0 + } + } + } + /// Called from task thread pub(crate) unsafe fn callback(task: *mut thread_pool::Task) { // SAFETY: task points to Task.task field @@ -2212,12 +2330,14 @@ impl<'a> Installer<'a> { let manager = self.manager(); let target_cpu = manager.options.cpu; let target_os = manager.options.os; + let target_libc = manager.options.libc; if let Some(replacement_pkg_id) = postinstall_optimizer::PostinstallOptimizer::get_native_binlink_replacement_package_id( pkg_resolutions_lists[pkg_id as usize].get(pkg_resolutions_buffer), pkg_metas, target_cpu, target_os, + target_libc, ) { for (new_entry_id, new_node_id) in entry_node_ids.iter().enumerate() { @@ -2245,7 +2365,7 @@ impl<'a> Installer<'a> { &self, entry_id: StoreEntryId, strategy: symlinker::Strategy, - ) -> sys::Result { + ) -> core::result::Result { let lockfile = self.lockfile(); let string_buf = lockfile.buffers.string_bytes.as_slice(); let dependencies = lockfile.buffers.dependencies.as_slice(); @@ -2270,19 +2390,45 @@ impl<'a> Installer<'a> { let dep_name = dependencies[dep.dep_id as usize].name.slice(string_buf); dest.set_length(base_len); - let _ = dest.append(dep_name); // OOM/capacity: fire-and-forget - if entry_node_modules_name.is_some_and(|name| strings::eql_long(dep_name, name, true)) { - // same name as the entry itself: nest one node_modules deeper to avoid the collision - let _ = dest.append(b"node_modules"); // OOM/capacity: fire-and-forget - let _ = dest.append(dep_name); // OOM/capacity: fire-and-forget - } + append_dependency_link_name(&mut dest, dep_name, entry_node_modules_name); + + let dep_node_id = self.store.entries.items_node_id()[dep.entry_id.get() as usize]; + let dep_pkg_id = self.store.nodes.items_pkg_id()[dep_node_id.get() as usize]; + let dep_res = &pkg_resolutions[dep_pkg_id as usize]; + let folder_is_inside_this_package = dep_res.tag == ResolutionTag::Folder + && super::folder_is_inside_package(lockfile, pkg_id, dep_pkg_id); + let dep_is_nested_folder = + self.store.entries.items_nested_folder()[dep.entry_id.get() as usize]; + // `build_store` only binds a nested folder to the packages containing it. + debug_assert!(folder_is_inside_this_package || !dep_is_nested_folder); let mut dep_store_path = AutoAbsPath::init_top_level_dir(); - if uses_global_store { + if folder_is_inside_this_package { + let Some(package_dir_name) = entry_node_modules_name else { + continue; + }; + if !self + .append_nested_folder_path( + &mut dep_store_path, + entry_id, + package_dir_name, + dep_res.folder().slice(string_buf), + ) + .map_err(TaskError::SymlinkDependencies)? + { + continue; + } + } else if dep_is_nested_folder { + continue; + } else if uses_global_store { debug_assert!(self.entry_uses_global_store(dep.entry_id)); self.append_real_store_path(&mut dep_store_path, dep.entry_id, Which::Final); - } else { - self.append_store_path(&mut dep_store_path, dep.entry_id); + } else if self + .append_dependency_path(&mut dep_store_path, dep.entry_id) + .is_err() + { + // The `link:` entry itself reports the over-long target. + continue; } let dest_len = dest.len(); @@ -2298,13 +2444,67 @@ impl<'a> Installer<'a> { }; let result = symlinker.ensure_symlink(strategy); dest = symlinker.dest.into_sep::<{ PathSeparators::AUTO }>(); - changed |= result?; + changed |= result.map_err(TaskError::SymlinkDependencies)?; } Ok(changed) } - pub(crate) fn link_dependency_bins(&self, parent_entry_id: StoreEntryId) -> crate::Result<()> { + /// Main thread, once every task is done: dependents may still be linking while a script fails. + pub(crate) fn unlink_failed_optional_entries(&self) { + if self.failed_optional_entries.is_empty() { + return; + } + + let lockfile = self.lockfile(); + let string_buf = lockfile.buffers.string_bytes.as_slice(); + let dependencies = lockfile.buffers.dependencies.as_slice(); + let pkg_names = lockfile.packages.items_name(); + let pkg_resolutions = lockfile.packages.items_resolution(); + + let entry_node_ids = self.store.entries.items_node_id(); + let nodes = &self.store.nodes; + + // Not `parents`: public hoisting adds root dependencies without recording a parent. + for (entry_index, entry_deps) in self.store.entries.items_dependencies().iter().enumerate() + { + for dep in entry_deps.slice() { + if !self.failed_optional_entries.contains(&dep.entry_id) { + continue; + } + let entry_id = StoreEntryId::from(u32::try_from(entry_index).expect("int cast")); + let node_id = entry_node_ids[entry_index].get() as usize; + let pkg_id = nodes.items_pkg_id()[node_id]; + let entry_node_modules_name = self.entry_store_node_modules_package_name( + nodes.items_dep_id()[node_id], + pkg_id, + &pkg_resolutions[pkg_id as usize], + pkg_names, + ); + let mut dest = AutoPath::init_top_level_dir(); + self.append_real_store_node_modules_path(&mut dest, entry_id, Which::Final); + let dep_name = dependencies[dep.dep_id as usize].name.slice(string_buf); + append_dependency_link_name(&mut dest, dep_name, entry_node_modules_name); + let _ = remove_link(dest.slice_z()); + } + } + + for &entry_id in &self.failed_optional_entries { + if self.store.entries.items_hoisted()[entry_id.get() as usize] { + let node_id = entry_node_ids[entry_id.get() as usize].get() as usize; + let pkg_name = pkg_names[nodes.items_pkg_id()[node_id] as usize].slice(string_buf); + let mut hidden = AutoPath::init(); + let _ = hidden.append_fmt(format_args!("{NODE_MODULES_BUN}/node_modules")); + let _ = hidden.append(pkg_name); + let _ = remove_link(hidden.slice_z()); + } + } + } + + pub(crate) fn link_dependency_bins( + &self, + parent_entry_id: StoreEntryId, + ) -> core::result::Result<(), TaskError> { let lockfile = self.lockfile(); let store = self.store; @@ -2331,6 +2531,7 @@ impl<'a> Installer<'a> { let mut link_rel_buf = paths::path_buffer_pool::get(); let mut seen: StringHashMap<()> = StringHashMap::default(); + let mut failed = Vec::new(); let mut node_modules_path = DefaultAbsPath::init_top_level_dir(); self.append_real_store_node_modules_path( @@ -2414,6 +2615,8 @@ impl<'a> Installer<'a> { { bin_linker.target_node_modules_path = bin_linker.node_modules_path; bin_linker.target_package_name = package_name; + bin_linker.err = None; + bin_linker.skipped_due_to_missing_bin = false; if self.manager().options.log_level.is_verbose() { bun_core::pretty_errorln!( @@ -2427,11 +2630,14 @@ impl<'a> Installer<'a> { } if let Some(err) = bin_linker.err { - return Err(err); + failed.push((dep.entry_id, err)); } } - Ok(()) + if failed.is_empty() { + return Ok(()); + } + Err(TaskError::DependencyBinaries(failed.into_boxed_slice())) } /// True when this entry should live in the shared global virtual store @@ -2544,6 +2750,17 @@ impl<'a> Installer<'a> { } } + /// Renames a project-local entry's fully linked `StagingPath` onto its final + /// path, which is what the next install's skip check looks at. + pub(crate) fn commit_local_store_package(&self, entry_id: StoreEntryId) -> sys::Result<()> { + debug_assert!(!self.entry_uses_global_store(entry_id)); + let mut staging = AutoPath::init_top_level_dir(); + self.append_real_store_path(&mut staging, entry_id, Which::Staging); + let mut final_ = AutoPath::init_top_level_dir(); + self.append_real_store_path(&mut final_, entry_id, Which::Final); + rename_staging_into_place(Fd::cwd(), staging.slice_z(), final_.slice_z()) + } + /// Project-local path `node_modules/.bun/` (the symlink that /// points at the global virtual-store entry). Relative to top-level dir. pub(crate) fn append_local_store_entry_path( @@ -2619,16 +2836,7 @@ impl<'a> Installer<'a> { }; if is_symlink { - #[cfg(windows)] - { - if sys::rmdir(dest.slice_z()).err().is_some() { - let _ = sys::unlink(dest.slice_z()); - } - } - #[cfg(not(windows))] - { - let _ = sys::unlink(dest.slice_z()); - } + let _ = remove_link(dest.slice_z()); } else { let _ = Fd::cwd().delete_tree(dest.slice()); } @@ -2713,10 +2921,58 @@ impl<'a> Installer<'a> { buf.append(pkg_name.slice(string_buf)); return; } - self.append_store_path(buf, entry_id); + self.append_store_path_at(buf, entry_id, which); } + /// `append_store_path` for the entry a dependency symlink points at, the one place a + /// `link:` package's path (`/`) is needed. A target + /// that does not fit a path buffer fails with `LinkPathTooLong`. + pub(crate) fn append_dependency_path( + &self, + buf: &mut AutoAbsPath, + entry_id: StoreEntryId, + ) -> core::result::Result<(), TaskError> { + let node_id = self.store.entries.items_node_id()[entry_id.get() as usize]; + let pkg_id = self.store.nodes.items_pkg_id()[node_id.get() as usize]; + let pkg_res = self.lockfile().packages.items_resolution()[pkg_id as usize]; + + if pkg_res.tag != ResolutionTag::Symlink { + self.append_store_path(buf, entry_id); + return Ok(()); + } + + // Ensured on the main thread (`install_isolated_packages`) before any task starts; + // this runs on worker threads and cannot lazily init it. + let link_dir_path: &[u8] = &self.manager().global_link_dir_path; + debug_assert!(!link_dir_path.is_empty()); + let string_buf = self.lockfile().buffers.string_bytes.as_slice(); + let link_target = pkg_res.symlink().slice(string_buf); + + let mut join_buf = paths::path_buffer_pool::get(); + let Some(target) = paths::resolve_path::join_abs_string_buf_checked::( + link_dir_path, + &mut join_buf[..], + &[link_target], + ) else { + return Err(TaskError::LinkPathTooLong); + }; + + paths::PathLike::clear(buf); + buf.append(target).assume_ok(); // bounded by the same buffer size + Ok(()) + } + + /// `entry_id` must not be a `link:` package; see `append_dependency_path`. pub(crate) fn append_store_path(&self, buf: &mut impl paths::PathLike, entry_id: StoreEntryId) { + self.append_store_path_at(buf, entry_id, Which::Final); + } + + fn append_store_path_at( + &self, + buf: &mut impl paths::PathLike, + entry_id: StoreEntryId, + which: Which, + ) { let string_buf = self.lockfile().buffers.string_bytes.as_slice(); let entries = &self.store.entries; @@ -2762,21 +3018,7 @@ impl<'a> Installer<'a> { buf.append(pkg_res.workspace().slice(string_buf)); } ResolutionTag::Symlink => { - // Lazily ensuring the global link dir would mutate - // `*PackageManager`, but `append_store_path` is - // `&self` and may run on worker - // threads, so the lazy init is hoisted to the main-thread caller - // (`isolated_install::install_packages`, before any `start_task`). - // Reading the cached field here is then equivalent. - let symlink_dir_path: &[u8] = &self.manager().global_link_dir_path; - debug_assert!( - !symlink_dir_path.is_empty(), - "global_link_dir_path must be ensured before tasks start", - ); - - buf.clear(); - buf.append(symlink_dir_path); - buf.append(pkg_res.symlink().slice(string_buf)); + unreachable!("link: packages have no store path; see append_dependency_path") } _ => { let pkg_name = pkg_names[pkg_id as usize]; @@ -2786,11 +3028,47 @@ impl<'a> Installer<'a> { store::entry::fmt_store_path(entry_id, self.store, self.lockfile()), )); buf.append(b"node_modules"); - buf.append(pkg_name.slice(string_buf)); + match which { + Which::Final => buf.append(pkg_name.slice(string_buf)), + Which::Staging => { + buf.append_fmt(format_args!("{}", StagingPath(pkg_name.slice(string_buf)))) + } + } } } } + /// Appends `//` to `buf`. + /// `Ok(false)`: nothing to link (the path leaves the package, or no such directory). + fn append_nested_folder_path( + &self, + buf: &mut AutoAbsPath, + entry_id: StoreEntryId, + package_dir_name: &[u8], + folder: &[u8], + ) -> sys::Result { + if bin_real::bin_target_escapes_package_dir(folder) { + return Ok(false); + } + + self.append_real_store_node_modules_path(buf, entry_id, Which::Staging); + buf.append(package_dir_name).assume_ok(); + // `folder` comes from a third-party manifest, so its length is unchecked. + if buf.len() + 1 + folder.len() >= paths::MAX_PATH_BYTES { + return Err(sys::Error::from_code( + sys::Errno::ENAMETOOLONG, + sys::Tag::fstatat, + )); + } + buf.append_join(folder).assume_ok(); + + match sys::directory_exists_at(Fd::cwd(), buf.slice_z()) { + Ok(is_dir) => Ok(is_dir), + Err(err) if err.get_errno() == sys::Errno::ENOTDIR => Ok(false), + Err(err) => Err(err), + } + } + /// The directory name for the entry store node_modules install /// folder. /// ./node_modules/.bun/jquery@3.7.1/node_modules/jquery @@ -2830,12 +3108,36 @@ pub enum Which { /// The published location (`/links/`). Use for symlink /// *targets* that point at other entries, and for the warm-hit check. Final, - /// The per-process temp sibling (`.tmp-`) the build - /// steps write into. Use for *destinations* of clonefile/hardlink/ - /// dep-symlink/bin-link when building this entry. + /// What the build steps write into, renamed onto `Final` once complete: the + /// whole entry (`.tmp-`) for a global-store entry, only the + /// package directory (`StagingPath`) for a project-local one. Staging, } +fn append_dependency_link_name( + dest: &mut AutoPath, + dep_name: &[u8], + entry_node_modules_name: Option<&[u8]>, +) { + let _ = dest.append(dep_name); // OOM/capacity: fire-and-forget + if entry_node_modules_name.is_some_and(|name| strings::eql_long(dep_name, name, true)) { + // same name as the entry itself: nest one node_modules deeper to avoid the collision + let _ = dest.append(b"node_modules"); // OOM/capacity: fire-and-forget + let _ = dest.append(dep_name); // OOM/capacity: fire-and-forget + } +} + +/// Removes a symlink; on Windows also a directory symlink or junction (dangling or not). +pub(crate) fn remove_link(path: &ZStr) -> sys::Result<()> { + #[cfg(windows)] + { + if sys::rmdir(path).is_ok() { + return Ok(()); + } + } + sys::unlink(path) +} + fn is_rename_collision(err: &sys::Error) -> bool { match err.get_errno() { sys::Errno::EEXIST | sys::Errno::ENOTEMPTY => true, diff --git a/src/install/isolated_install/Store.rs b/src/install/isolated_install/Store.rs index b1a3bef316a2..27c5fbff18e8 100644 --- a/src/install/isolated_install/Store.rs +++ b/src/install/isolated_install/Store.rs @@ -7,7 +7,6 @@ use bstr::BStr; use bun_alloc::AllocError; use bun_collections::{ArrayHashMap, MultiArrayList}; use bun_semver::String as SemverString; -use bun_wyhash::Wyhash; use crate::lockfile::{Lockfile, package}; use crate::{Dependency, DependencyID, INVALID_DEPENDENCY_ID, PackageID, Resolution}; @@ -284,6 +283,10 @@ pub mod entry { // if true this entry gets symlinked to `node_modules/.bun/node_modules` pub hoisted: bool, + /// Exists only inside the packages declaring it (`isolated_install::contained_folders`): + /// no store directory, task or hoist slot; linked in `Installer::symlink_dependencies`. + pub nested_folder: bool, + pub peer_hash: PeerHash, /// Content hash of (package + sorted resolved dependency global-store keys), @@ -314,6 +317,7 @@ pub mod entry { parents: Vec, step: core::sync::atomic::AtomicU32, hoisted: bool, + nested_folder: bool, peer_hash: PeerHash, entry_hash: u64, scripts: core::cell::Cell>, @@ -329,6 +333,7 @@ pub mod entry { // `Step::LinkPackage as u32 == 0`. step: core::sync::atomic::AtomicU32::new(0), hoisted: false, + nested_folder: false, peer_hash: PeerHash::NONE, entry_hash: 0, scripts: core::cell::Cell::new(None), @@ -352,53 +357,30 @@ pub mod entry { } } - /// Bounds the entry name so the lifecycle-script cwd fits Windows' MAX_PATH; 80 keeps versions and `github+owner+repo+` verbatim. - const MAX_RESOLUTION_LEN: usize = 80; - /// Longer resolutions become `+<16 hex wyhash of the whole text>`, `MAX_RESOLUTION_LEN` bytes at most. - const CUT_RESOLUTION_LEN: usize = MAX_RESOLUTION_LEN - "+".len() - 16; - - /// The first `MAX_RESOLUTION_LEN` bytes written, plus the length and hash of all of them. - struct ResolutionSink { - buf: [u8; MAX_RESOLUTION_LEN], - len: usize, - hasher: Wyhash, - } - - impl fmt::Write for ResolutionSink { - fn write_str(&mut self, s: &str) -> fmt::Result { - let bytes = s.as_bytes(); - if let Some(room) = self.buf.get_mut(self.len..) { - let n = bytes.len().min(room.len()); - room[..n].copy_from_slice(&bytes[..n]); - } - self.len += bytes.len(); - self.hasher.update(bytes); - Ok(()) - } - } - + /// The text after `name@` in an entry name, capped at 80 bytes because the package directory + /// below is the cwd of lifecycle scripts and Windows' `CreateProcess` rejects one past + /// MAX_PATH. Versions and `github+owner+repo+` fit; a longer folder path or URL + /// becomes `+<16 hex wyhash of the whole text>`. fn write_resolution(f: &mut fmt::Formatter<'_>, resolution: fmt::Arguments<'_>) -> fmt::Result { - let mut sink = ResolutionSink { - buf: [0; MAX_RESOLUTION_LEN], - len: 0, - hasher: Wyhash::init(0), - }; - fmt::write(&mut sink, resolution)?; - - if sink.len <= MAX_RESOLUTION_LEN { - return f.write_str(bun_core::str_utf8(&sink.buf[..sink.len]).ok_or(fmt::Error)?); + const MAX_RESOLUTION_LEN: usize = 80; + let mut buf = [0u8; MAX_RESOLUTION_LEN]; + if let Ok(text) = bun_core::fmt::buf_print(&mut buf, resolution) { + return f.write_str(bun_core::str_utf8(text).ok_or(fmt::Error)?); } - - let mut cut = CUT_RESOLUTION_LEN; - while !bun_core::strings::is_on_char_boundary(&sink.buf, cut) { + let text = resolution.to_string(); + let mut cut = MAX_RESOLUTION_LEN - "+".len() - 16; + while !text.is_char_boundary(cut) { cut -= 1; } - f.write_str(bun_core::str_utf8(&sink.buf[..cut]).ok_or(fmt::Error)?)?; - write!(f, "+{:016x}", sink.hasher.final_()) + write!( + f, + "{}+{:016x}", + &text[..cut], + bun_wyhash::hash(text.as_bytes()) + ) } /// `name@version` (or `name@file+path` / `name@root`) without the `+peerhash` suffix. - /// The resolution part is bounded by [`MAX_RESOLUTION_LEN`]. pub struct StoreKeyFormatter<'a> { name: SemverString, resolution: &'a Resolution, diff --git a/src/install/isolated_install/Symlinker.rs b/src/install/isolated_install/Symlinker.rs index 52c90735abba..f4cdcda503a2 100644 --- a/src/install/isolated_install/Symlinker.rs +++ b/src/install/isolated_install/Symlinker.rs @@ -76,7 +76,7 @@ impl Symlinker { }, }, // readlink failed for a reason other than NOENT — - // dest exists but isn't a symlink. If it's a real + // dest exists but isn't a symlink. If it's a non-empty // directory, leave it: this is the `bun patch ` // workspace (a detached copy the user is editing // before `--commit`), and `deleteTree` here would @@ -100,9 +100,13 @@ impl Symlinker { false }; if is_dir { - return Ok(false); + // rmdir succeeds only on an empty directory. + if bun_sys::rmdir(self.dest.slice_z()).is_err() { + return Ok(false); + } + } else { + let _ = bun_sys::unlink(self.dest.slice_z()); } - let _ = bun_sys::unlink(self.dest.slice_z()); return self.symlink().map(|()| true); } }; diff --git a/src/install/lib.rs b/src/install/lib.rs index aaa1d4ba102f..5561917e4411 100644 --- a/src/install/lib.rs +++ b/src/install/lib.rs @@ -857,9 +857,13 @@ impl RunCommand { } } - // DirInfo walk / npm_package_* seeding is performed by the T6 impl - // (`bun_runtime::cli::RunCommand::configure_env_for_run`); install - // callers discard the return value. + // Overwrite like npm: an outer `bun run` in another project may have left a stale value. + env_loader.map.put( + b"npm_config_local_prefix", + bun_fs::FileSystem::instance().top_level_dir(), + )?; + + // `npm_package_*` is per package; see `spawn_package_lifecycle_scripts`. Ok(core::ptr::null_mut()) } } diff --git a/src/install/lifecycle_script_runner.rs b/src/install/lifecycle_script_runner.rs index 463fcea10407..77c90fff9b39 100644 --- a/src/install/lifecycle_script_runner.rs +++ b/src/install/lifecycle_script_runner.rs @@ -247,6 +247,33 @@ pub fn replace_package_manager_run( Ok(()) } +/// ` -c