From 0343356c0e6d6642fcea25d7d61fa608a4f8bc74 Mon Sep 17 00:00:00 2001 From: ZKulbeda Date: Sun, 13 Aug 2023 23:04:18 +0000 Subject: [PATCH 1/5] add workflow --- .github/workflows/benchmark.yml | 96 +++++++++++ library/src/benchmarks/compare-and-format.mjs | 151 ++++++++++++++++++ 2 files changed, 247 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 library/src/benchmarks/compare-and-format.mjs diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..06e6d0531 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,96 @@ +name: Perfomance changes + +on: + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install Node.js + uses: actions/setup-node@v3 + with: + node-version: 16 + + - uses: pnpm/action-setup@v2 + name: Install pnpm + id: pnpm-install + with: + version: 8 + run_install: false + + - name: Get pnpm store directory + id: pnpm-cache + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + + - uses: actions/cache@v3 + name: Setup pnpm cache + with: + path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store- + + - name: Install dependencies + run: pnpm install + + - name: Get after commit sha + id: after_commit_sha + run: echo "::set-output name=sha::$(git rev-parse --short HEAD)" + + - name: Run Vitest after benchmark + run: cd ./library && pnpm bench --run --reporter json --outputFile /tmp/after.json + + - name: Checkout main branche + uses: actions/checkout@v2 + with: + ref: main + + - name: Get before commit sha + id: before_commit_sha + run: echo "::set-output name=sha::$(git rev-parse --short HEAD)" + + - name: Run Vitest before benchmark + run: cd ./library && pnpm bench --run --reporter json --outputFile /tmp/before.json + + - name: Get current job log URL + uses: Tiryoh/gha-jobid-action@v0 + id: job-url + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + job_name: '${{ github.job }}' + + - name: Compare results and format message + id: compare + uses: actions/github-script@v6 + with: + result-encoding: string + script: | + const { generateMessage } = await import( + './library/src/benchmark/compare-and-format.mjs' + ); + const fs = await import('node:fs/promises'); + const before = JSON.parse(await fs.readFile('/tmp/before.json', 'utf-8')); + const after = JSON.parse(await fs.readFile('/tmp/after.json', 'utf-8')); + + return generateMessage(before, after, { + beforeSha: '${{steps.before_commit_sha.outputs.sha}}', + afterSha: '${{steps.after_commit_sha.outputs.sha}}', + beforeBenchStepLink: '${{steps.job-url.outputs.html_url}}#step:11:1', + afterBenchStepLink: '${{steps.job-url.outputs.html_url}}#step:8:1', + repoLink: '${{github.server_url}}/${{ github.repository }}', + }); + + - name: Leave sticky comment + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: Performance changes + message: ${{ steps.compare.outputs.result }} diff --git a/library/src/benchmarks/compare-and-format.mjs b/library/src/benchmarks/compare-and-format.mjs new file mode 100644 index 000000000..573dca4fd --- /dev/null +++ b/library/src/benchmarks/compare-and-format.mjs @@ -0,0 +1,151 @@ +/** + * + * @param {BenchmarkSuiteResult} before + * @param {BenchmarkSuiteResult} after + * @param {object} config + * @param {string} config.beforeSha + * @param {string} config.afterSha + * @param {string} config.beforeBenchStepLink + * @param {string} config.afterBenchStepLink + * @param {string} config.repoLink + */ +export function generateMessage(before, after, config) { + const table = compare(before, after); + const stringTable = generateMarkdownTable(table, [ + ['desc', 'description', 'left'], + ['name', 'name', 'center'], + ['beforeHz', 'before, hz', 'right'], + ['beforeRme', 'rme', 'left'], + ['afterHz', 'after, hz', 'right'], + ['afterRme', 'rme', 'left'], + ['diff', 'diff', 'center'], + ]); + + const beforeRefLink = `${config.repoLink}/commit/${config.beforeSha}`; + const afterRefLink = `${config.repoLink}/commit/${config.afterSha}`; + + return `Measured benckmark chages between [${config.beforeSha}](${beforeRefLink}) and [${config.afterSha}](${afterRefLink}). +
+Show table + +${stringTable} + +> [!NOTE] +> **hz** – the number of operations per second (higher – better) +> **rme** – relative margin of error + +
+ +Full log and details of [before](${config.beforeBenchStepLink}) and [after](${config.afterBenchStepLink}) benchmark lunch.`; +} + +const hzFormatter = Intl.NumberFormat('en-US', { + maximumFractionDigits: 2, +}); + +const rmeFormatter = Intl.NumberFormat('en-US', { + maximumFractionDigits: 2, + style: 'percent', +}); + +const diffFormatter = Intl.NumberFormat('en-US', { + maximumFractionDigits: 2, + style: 'percent', +}); + +/** + * + * @param {BenchmarkSuiteResult} before + * @param {BenchmarkSuiteResult} after + */ +export const compare = (before, after) => { + const beforeResults = before.testResults; + const afterResults = after.testResults; + + const table = []; + + for (const [description, cases] of Object.entries(afterResults)) { + cases.sort((a, b) => a.rank - b.rank); + for (const [index, afterCase] of cases.entries()) { + const beforeCase = beforeResults[description].find( + (bench) => bench.name === afterCase.name + ); + const row = { + desc: index === 0 ? description : '', + name: afterCase.name, + afterHz: hzFormatter.format(afterCase.hz), + afterRme: '±' + rmeFormatter.format(afterCase.rme / 100), + }; + if (beforeCase) { + row.beforeHz = hzFormatter.format(beforeCase.hz); + row.beforeRme = '±' + rmeFormatter.format(beforeCase.rme / 100); + + const diffValue = (afterCase.hz - beforeCase.hz) / beforeCase.hz; + row.diff = diffFormatter.format(diffValue); + if (diffValue < 0) { + row.diff = `**${row.diff}**`; + } + } + table.push(row); + } + // empty row as a separator in table + table.push({}); + } + // remove last empty row + table.pop(); + return table; +}; + +/** + * + * @param {object[]} objects + * @param {Array[string, string, "left" | "right" | "center"]} fields + * @returns + */ +export function generateMarkdownTable(objects, columns) { + const alignMap = { + left: ':---', + right: '---:', + center: ':---:', + }; + const headers = columns.map(([, header]) => `${header} |`); + const aligns = columns.map(([, , align]) => `${alignMap[align]} |`); + let table = `| ${headers.join(' ')}\n| ${aligns.join(' ')}\n`; + objects.forEach((obj) => { + let row = columns.map(([field]) => obj[field]); + table += `| ${row.map((value) => `${value ?? ''} |`).join(' ')}\n`; + }); + return table; +} + +/** + * @typedef {Object} BenchmarkResult + * @property {string} name + * @property {number} rank + * @property {number} rme + * @property {number} totalTime + * @property {number} min + * @property {number} max + * @property {number} hz + * @property {number} period + * @property {number} mean + * @property {number} variance + * @property {number} sd + * @property {number} sem + * @property {number} df + * @property {number} critical + * @property {number} moe + * @property {number} p75 + * @property {number} p99 + * @property {number} p995 + * @property {number} p999 + */ + +/** + * An object representing the results of a test suite. + * + * @typedef {Object} BenchmarkSuiteResult + * @property {number} numTotalTestSuites + * @property {number} numTotalTests + * @property {Object.>} testResults + */ From f01ade3bcd2256a20ab247810db28509be3dedab Mon Sep 17 00:00:00 2001 From: ZKulbeda Date: Sun, 13 Aug 2023 23:45:36 +0000 Subject: [PATCH 2/5] `Measured performance` in comment --- library/src/benchmarks/compare-and-format.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/src/benchmarks/compare-and-format.mjs b/library/src/benchmarks/compare-and-format.mjs index 573dca4fd..53a8ceb5c 100644 --- a/library/src/benchmarks/compare-and-format.mjs +++ b/library/src/benchmarks/compare-and-format.mjs @@ -24,7 +24,7 @@ export function generateMessage(before, after, config) { const beforeRefLink = `${config.repoLink}/commit/${config.beforeSha}`; const afterRefLink = `${config.repoLink}/commit/${config.afterSha}`; - return `Measured benckmark chages between [${config.beforeSha}](${beforeRefLink}) and [${config.afterSha}](${afterRefLink}). + return `Measured performance chages between [${config.beforeSha}](${beforeRefLink}) and [${config.afterSha}](${afterRefLink}).
Show table From c1949a9a9fa79a74308bec860a01a517b366cf54 Mon Sep 17 00:00:00 2001 From: ZKulbeda Date: Sun, 13 Aug 2023 23:55:53 +0000 Subject: [PATCH 3/5] make checkouts action's version equals --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 06e6d0531..f4c3c9fbe 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -50,7 +50,7 @@ jobs: run: cd ./library && pnpm bench --run --reporter json --outputFile /tmp/after.json - name: Checkout main branche - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: main From 2d89daf27d6ed14c362c36c04d21ff9a00bafee4 Mon Sep 17 00:00:00 2001 From: ZKulbeda Date: Sun, 13 Aug 2023 23:58:04 +0000 Subject: [PATCH 4/5] fix typo --- library/src/benchmarks/compare-and-format.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/src/benchmarks/compare-and-format.mjs b/library/src/benchmarks/compare-and-format.mjs index 53a8ceb5c..b8d92b55f 100644 --- a/library/src/benchmarks/compare-and-format.mjs +++ b/library/src/benchmarks/compare-and-format.mjs @@ -24,7 +24,7 @@ export function generateMessage(before, after, config) { const beforeRefLink = `${config.repoLink}/commit/${config.beforeSha}`; const afterRefLink = `${config.repoLink}/commit/${config.afterSha}`; - return `Measured performance chages between [${config.beforeSha}](${beforeRefLink}) and [${config.afterSha}](${afterRefLink}). + return `Measured performance changes between [${config.beforeSha}](${beforeRefLink}) and [${config.afterSha}](${afterRefLink}).
Show table From ffcb46a7466cecc74f19916fc5cc68c076f35143 Mon Sep 17 00:00:00 2001 From: ZKulbeda Date: Sun, 13 Aug 2023 23:59:35 +0000 Subject: [PATCH 5/5] fix typo --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f4c3c9fbe..d09a7f480 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -49,7 +49,7 @@ jobs: - name: Run Vitest after benchmark run: cd ./library && pnpm bench --run --reporter json --outputFile /tmp/after.json - - name: Checkout main branche + - name: Checkout main branch uses: actions/checkout@v3 with: ref: main