Skip to content

Commit c44f016

Browse files
authored
Implements a benchmarking script (#5536)
**What's the problem this PR addresses?** Making benchmarks required a little too much setup for me, so I made a script to make it seamless. **How did you fix it?** Running `yarn bench` in the repository will spawn a shell inside a temporary folder, with everything setup to run a benchmark when calling `bench-run`. It doesn't handle yet parametrized benchmarks, but should support comparing performances before/after. Incidentally, the `yarn set version from sources` command now caches the result, to avoid rebuilding the same commits. **Checklist** <!--- Don't worry if you miss something, chores are automatically tested. --> <!--- This checklist exists to help you remember doing the chores when you submit a PR. --> <!--- Put an `x` in all the boxes that apply. --> - [x] I have read the [Contributing Guide](https://yarnpkg.com/advanced/contributing). <!-- See https://yarnpkg.com/advanced/contributing#preparing-your-pr-to-be-released for more details. --> <!-- Check with `yarn version check` and fix with `yarn version check -i` --> - [x] I have set the packages that need to be released for my changes to be effective. <!-- The "Testing chores" workflow validates that your PR follows our guidelines. --> <!-- If it doesn't pass, click on it to see details as to what your PR might be missing. --> - [x] I will check that all automated PR checks pass before the PR gets reviewed.
1 parent edf82db commit c44f016

File tree

6 files changed

+194
-11
lines changed

6 files changed

+194
-11
lines changed

.pnp.cjs

+2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

.yarn/versions/04b94c40.yml

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
releases:
2+
"@yarnpkg/cli": patch
3+
"@yarnpkg/plugin-essentials": patch
4+
5+
declined:
6+
- "@yarnpkg/plugin-compat"
7+
- "@yarnpkg/plugin-constraints"
8+
- "@yarnpkg/plugin-dlx"
9+
- "@yarnpkg/plugin-init"
10+
- "@yarnpkg/plugin-interactive-tools"
11+
- "@yarnpkg/plugin-nm"
12+
- "@yarnpkg/plugin-npm-cli"
13+
- "@yarnpkg/plugin-pack"
14+
- "@yarnpkg/plugin-patch"
15+
- "@yarnpkg/plugin-pnp"
16+
- "@yarnpkg/plugin-pnpm"
17+
- "@yarnpkg/plugin-stage"
18+
- "@yarnpkg/plugin-typescript"
19+
- "@yarnpkg/plugin-version"
20+
- "@yarnpkg/plugin-workspace-tools"
21+
- "@yarnpkg/builder"
22+
- "@yarnpkg/core"
23+
- "@yarnpkg/doctor"

package.json

+4
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
}
6767
},
6868
"scripts": {
69+
"bench": "yarn node -r ./scripts/setup-ts-execution ./scripts/bench-folder.ts",
6970
"build:plugin-constraints": "yarn node -r ./scripts/setup-ts-execution ./scripts/create-mock-plugin.ts constraints",
7071
"build:plugin-exec": "yarn node -r ./scripts/setup-ts-execution ./scripts/create-mock-plugin.ts exec",
7172
"build:plugin-interactive-tools": "yarn node -r ./scripts/setup-ts-execution ./scripts/create-mock-plugin.ts interactive-tools",
@@ -91,5 +92,8 @@
9192
},
9293
"engines": {
9394
"node": ">=18.12.0"
95+
},
96+
"dependencies": {
97+
"chalk": "^3.0.0"
9498
}
9599
}

packages/plugin-essentials/sources/commands/set/version/sources.ts

+21-11
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,12 @@ const cloneWorkflow = ({repository, branch}: {repository: string, branch: string
2828
const updateWorkflow = ({branch}: {branch: string}) => [
2929
[`git`, `fetch`, `origin`, `--depth=1`, getBranchRef(branch), `--force`],
3030
[`git`, `reset`, `--hard`, `FETCH_HEAD`],
31-
[`git`, `clean`, `-dfx`],
31+
[`git`, `clean`, `-dfx`, `-e`, `packages/yarnpkg-cli/bundles`],
3232
];
3333

34-
const buildWorkflow = ({plugins, noMinify}: {noMinify: boolean, plugins: Array<string>}, target: PortablePath) => [
34+
const buildWorkflow = ({plugins, noMinify}: {noMinify: boolean, plugins: Array<string>}, output: PortablePath, target: PortablePath) => [
3535
[`yarn`, `build:cli`, ...new Array<string>().concat(...plugins.map(plugin => [`--plugin`, ppath.resolve(target, plugin as Filename)])), ...noMinify ? [`--no-minify`] : [], `|`],
36+
[`mv`, `packages/yarnpkg-cli/bundles/yarn.js`, npath.fromPortablePath(output), `|`],
3637
];
3738

3839
// eslint-disable-next-line arca/no-default-export
@@ -70,6 +71,10 @@ export default class SetVersionSourcesCommand extends BaseCommand {
7071
description: `An array of additional plugins that should be included in the bundle`,
7172
});
7273

74+
dryRun = Option.Boolean(`-n,--dry-run`, false, {
75+
description: `If set, the bundle will be built but not added to the project`,
76+
});
77+
7378
noMinify = Option.Boolean(`--no-minify`, false, {
7479
description: `Build a bundle for development (debugging) - non-minified and non-mangled`,
7580
});
@@ -100,19 +105,24 @@ export default class SetVersionSourcesCommand extends BaseCommand {
100105
report.reportInfo(MessageName.UNNAMED, `Building a fresh bundle`);
101106
report.reportSeparator();
102107

103-
await runWorkflow(buildWorkflow(this, target), {configuration, context: this.context, target});
108+
const commitHash = await execUtils.execvp(`git`, [`rev-parse`, `--short`, `HEAD`], {cwd: target, strict: true});
109+
const bundlePath = ppath.join(target, `packages/yarnpkg-cli/bundles/yarn-${commitHash.stdout.trim()}.js`);
104110

105-
report.reportSeparator();
111+
if (!xfs.existsSync(bundlePath)) {
112+
await runWorkflow(buildWorkflow(this, bundlePath, target), {configuration, context: this.context, target});
113+
report.reportSeparator();
114+
}
106115

107-
const bundlePath = ppath.resolve(target, `packages/yarnpkg-cli/bundles/yarn.js`);
108116
const bundleBuffer = await xfs.readFilePromise(bundlePath);
109117

110-
const {bundleVersion} = await setVersion(configuration, null, async () => bundleBuffer, {
111-
report,
112-
});
118+
if (!this.dryRun) {
119+
const {bundleVersion} = await setVersion(configuration, null, async () => bundleBuffer, {
120+
report,
121+
});
113122

114-
if (!this.skipPlugins) {
115-
await updatePlugins(this, bundleVersion, {project, report, target});
123+
if (!this.skipPlugins) {
124+
await updatePlugins(this, bundleVersion, {project, report, target});
125+
}
116126
}
117127
});
118128

@@ -167,7 +177,7 @@ export async function prepareRepo(spec: PrepareSpec, {configuration, report, tar
167177
try {
168178
await runWorkflow(updateWorkflow(spec), {configuration, context: spec.context, target});
169179
ready = true;
170-
} catch (error) {
180+
} catch {
171181
report.reportSeparator();
172182
report.reportWarning(MessageName.UNNAMED, `Repository update failed; we'll try to regenerate it`);
173183
}

scripts/bench-folder.ts

+143
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
import {Filename, npath, ppath, xfs} from '@yarnpkg/fslib';
2+
import chalk from 'chalk';
3+
import cp from 'child_process';
4+
5+
const repoDir = ppath.dirname(npath.toPortablePath(__dirname));
6+
7+
xfs.mktempSync(temp => {
8+
const nextEnv = {...process.env};
9+
10+
nextEnv.YARN_IGNORE_PATH = `1`;
11+
nextEnv.NODE_OPTIONS = ``;
12+
13+
function exec(arg0: string, argv: Array<string>, opts: cp.ExecFileSyncOptions) {
14+
try {
15+
cp.execFileSync(arg0, argv, {cwd: npath.fromPortablePath(temp), env: nextEnv, ...opts});
16+
} catch (err) {
17+
if (err.status !== 128) {
18+
throw err;
19+
}
20+
}
21+
}
22+
23+
const binaryDir = ppath.join(temp, `bin`);
24+
xfs.mkdirSync(binaryDir);
25+
nextEnv.PATH = `${npath.fromPortablePath(binaryDir)}:${process.env.PATH}`;
26+
27+
// We create two binary folders; each benchmark will use one or the other
28+
xfs.mkdirSync(ppath.join(binaryDir, `before`));
29+
xfs.symlinkSync(ppath.join(binaryDir, `yarn-before`), ppath.join(binaryDir, `before`, `yarn`));
30+
xfs.symlinkSync(ppath.join(repoDir, `packages/yarnpkg-cli/bundles/yarn.js`), ppath.join(binaryDir, `yarn-after`));
31+
32+
xfs.mkdirSync(ppath.join(binaryDir, `after`));
33+
xfs.symlinkSync(ppath.join(binaryDir, `yarn-after`), ppath.join(binaryDir, `after`, `yarn`));
34+
35+
// We create a few helper scripts to make the benchmarking easier
36+
xfs.writeFileSync(ppath.join(binaryDir, `yarn-repo`), [
37+
`#!/bin/bash\n`,
38+
`cd ${npath.fromPortablePath(repoDir)}\n`,
39+
`unset YARN_IGNORE_PATH\n`,
40+
`export PATH="${process.env.PATH}"\n`,
41+
`exec yarn $@\n`,
42+
].join(``));
43+
44+
xfs.writeFileSync(ppath.join(binaryDir, `bench-commit`), [
45+
`#!/bin/bash\n`,
46+
`cd ${npath.fromPortablePath(temp)}\n`,
47+
`git add . && (git diff-index --quiet HEAD || git commit -m Commit > /dev/null)\n`,
48+
].join(``));
49+
50+
xfs.writeFileSync(ppath.join(binaryDir, `bench-reset`), [
51+
`#!/bin/bash\n`,
52+
`cd ${npath.fromPortablePath(temp)}\n`,
53+
`git reset --hard HEAD && git clean -fdx\n`,
54+
].join(``));
55+
56+
xfs.writeFileSync(ppath.join(binaryDir, `bench-run`), [
57+
`#!/bin/bash\n`,
58+
`cd ${npath.fromPortablePath(temp)}\n`,
59+
`\n`,
60+
`PATH="$(pwd)/bin/hyperfine:$PATH" hyperfine --warmup 1 \\\n`,
61+
` --export-markdown=.git/yarn-bench \\\n`,
62+
` --prepare 'bench-reset && bash "${npath.fromPortablePath(temp)}"/bench-prepare.sh' \\\n`,
63+
` 'before' \\\n`,
64+
` 'after'\n`,
65+
`\n`,
66+
`if which pbcopy >/dev/null 2>&1; then\n`,
67+
` pbcopy < .git/yarn-bench\n`,
68+
`fi\n`,
69+
].join(``));
70+
71+
// We don't want Yarn to be called directly, since it'd be a global copy we don't control
72+
xfs.writeFileSync(ppath.join(binaryDir, `yarn`), [
73+
`#!/bin/bash\n`,
74+
`echo "Don't run Yarn directly:"\n`,
75+
`echo\n`,
76+
`echo " - if the output is the same between both versions, use yarn-before or yarn-after instead"\n`,
77+
`echo " - otherwise, add the call to the bench-prepare.sh script"\n`,
78+
`exit 1\n`,
79+
].join(``));
80+
81+
// We also create another binary dir, just to clean up the command line we show in Hyperfine
82+
xfs.mkdirSync(ppath.join(binaryDir, `hyperfine`));
83+
84+
xfs.writeFileSync(ppath.join(binaryDir, `hyperfine/before`), [
85+
`#!/bin/bash\n`,
86+
`cd ${npath.fromPortablePath(temp)}\n`,
87+
`PATH="$(pwd)/bin/before:$PATH" bash bench-script.sh\n`,
88+
].join(``));
89+
90+
xfs.writeFileSync(ppath.join(binaryDir, `hyperfine/after`), [
91+
`#!/bin/bash\n`,
92+
`cd ${npath.fromPortablePath(temp)}\n`,
93+
`PATH="$(pwd)/bin/after:$PATH" bash bench-script.sh\n`,
94+
].join(``));
95+
96+
// Those two scripts are meant to be written by the user (but not run directly, so not executable and no shebang)
97+
xfs.writeFileSync(ppath.join(temp, `bench-prepare.sh`), ``);
98+
xfs.writeFileSync(ppath.join(temp, `bench-script.sh`), `yarn install\n`);
99+
100+
// General Yarn configuration
101+
xfs.writeJsonSync(ppath.join(temp, Filename.rc), {
102+
globalFolder: `.yarn/global`,
103+
});
104+
105+
xfs.writeJsonSync(ppath.join(temp, Filename.manifest), {
106+
name: `benchmark`,
107+
private: true,
108+
});
109+
110+
// We retrieve the latest Yarn version from master, and add it to the PATH
111+
exec(`yarn-after`, [`set`, `version`, `from`, `sources`], {stdio: `inherit`});
112+
113+
const releaseFolder = ppath.join(temp, `.yarn/releases`);
114+
xfs.moveSync(ppath.join(releaseFolder, xfs.readdirSync(releaseFolder)[0]), ppath.join(binaryDir, `yarn-before`));
115+
116+
// All binaries should be executable
117+
for (const name of xfs.readdirSync(binaryDir, {recursive: true}))
118+
xfs.chmodSync(ppath.join(binaryDir, name), 0o755);
119+
120+
exec(`git`, [`init`], {stdio: `ignore`});
121+
exec(`git`, [`config`, `core.hooksPath`, npath.fromPortablePath(temp)], {stdio: `ignore`});
122+
exec(`git`, [`add`, `.`], {stdio: `ignore`});
123+
exec(`git`, [`commit`, `-m`, `First commit`, `--allow-empty`], {stdio: `ignore`});
124+
exec(`bench-commit`, [], {stdio: `ignore`});
125+
126+
process.stdout.write(`\x1bc`);
127+
128+
console.log(`You're now in the benchmarking environment. Here is how it works:`);
129+
console.log();
130+
console.log(` - Setup the initial state of your repository, then run ${chalk.magenta(`bench-commit`)} to persist it in the temporary repository.`);
131+
console.log(` (note that this repository intentionally lacks a gitignore - feel free to commit archives, metadata, etc)`);
132+
console.log(` - If you want to run some code before the benchmark, add it to the ${chalk.yellow(`bench-prepare.sh`)} script in this folder.`);
133+
console.log(` - By default the benchmark will run ${chalk.magenta(`yarn install`)}; you can change that by editing ${chalk.yellow(`bench-script.sh`)}.`);
134+
console.log(` - When you want to run the benchmark, run ${chalk.magenta(`bench-run`)}. The repository will be reset between each run.`);
135+
console.log(` - If using OSX, the results will be automatically copied to your clipboard. Otherwise, they'll be available in ${chalk.yellow(`.git/yarn-bench`)}.`);
136+
console.log();
137+
console.log(`Once you're done, exit the shell and the temporary environment will be removed.`);
138+
139+
nextEnv.PS1 = `\\[\x1b[94m\\](Yarn benchmarking tool)\\[\x1b[39m\\] \\[\x1b[1m\\]$\\[\x1b[22m\\] `;
140+
nextEnv.PROMPT_COMMAND = `echo; trap 'echo; trap - DEBUG' DEBUG`;
141+
142+
exec(`bash`, [], {stdio: `inherit`});
143+
});

yarn.lock

+1
Original file line numberDiff line numberDiff line change
@@ -7351,6 +7351,7 @@ __metadata:
73517351
"@yarnpkg/fslib": "workspace:^"
73527352
"@yarnpkg/libzip": "workspace:^"
73537353
"@yarnpkg/sdks": "workspace:^"
7354+
chalk: "npm:^3.0.0"
73547355
clipanion: "npm:^3.2.1"
73557356
esbuild-wasm: "npm:0.17.5"
73567357
eslint: "npm:^8.2.0"

0 commit comments

Comments
 (0)