Add github actions workflow for perfomance comparison in pull requests

.github/workflows/benchmark.yml

@@ -0,0 +1,96 @@
+name: Perfomance changes
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - uses: pnpm/action-setup@v2
+        name: Install pnpm
+        id: pnpm-install
+        with:
+          version: 8
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache@v3
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Get after commit sha
+        id: after_commit_sha
+        run: echo "::set-output name=sha::$(git rev-parse --short HEAD)"
+
+      - name: Run Vitest after benchmark
+        run: cd ./library && pnpm bench --run --reporter json --outputFile /tmp/after.json
+
+      - name: Checkout main branch
+        uses: actions/checkout@v3
+        with:
+          ref: main
+
+      - name: Get before commit sha
+        id: before_commit_sha
+        run: echo "::set-output name=sha::$(git rev-parse --short HEAD)"
+
+      - name: Run Vitest before benchmark
+        run: cd ./library && pnpm bench --run --reporter json --outputFile /tmp/before.json
+
+      - name: Get current job log URL
+        uses: Tiryoh/gha-jobid-action@v0
+        id: job-url
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          job_name: '${{ github.job }}'
+
+      - name: Compare results and format message
+        id: compare
+        uses: actions/github-script@v6
+        with:
+          result-encoding: string
+          script: |
+            const { generateMessage } = await import(
+              './library/src/benchmark/compare-and-format.mjs'
+            );
+            const fs = await import('node:fs/promises');
+            const before = JSON.parse(await fs.readFile('/tmp/before.json', 'utf-8'));
+            const after = JSON.parse(await fs.readFile('/tmp/after.json', 'utf-8'));
+
+            return generateMessage(before, after, {
+              beforeSha: '${{steps.before_commit_sha.outputs.sha}}',
+              afterSha: '${{steps.after_commit_sha.outputs.sha}}',
+              beforeBenchStepLink: '${{steps.job-url.outputs.html_url}}#step:11:1',
+              afterBenchStepLink: '${{steps.job-url.outputs.html_url}}#step:8:1',
+              repoLink: '${{github.server_url}}/${{ github.repository }}',
+            });
+
+      - name: Leave sticky comment
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: Performance changes
+          message: ${{ steps.compare.outputs.result }}

library/src/benchmarks/compare-and-format.mjs

@@ -0,0 +1,151 @@
+/**
+ *
+ * @param {BenchmarkSuiteResult} before
+ * @param {BenchmarkSuiteResult} after
+ * @param {object} config
+ * @param {string} config.beforeSha
+ * @param {string} config.afterSha
+ * @param {string} config.beforeBenchStepLink
+ * @param {string} config.afterBenchStepLink
+ * @param {string} config.repoLink
+ */
+export function generateMessage(before, after, config) {
+  const table = compare(before, after);
+  const stringTable = generateMarkdownTable(table, [
+    ['desc', 'description', 'left'],
+    ['name', 'name', 'center'],
+    ['beforeHz', 'before, hz', 'right'],
+    ['beforeRme', 'rme', 'left'],
+    ['afterHz', 'after, hz', 'right'],
+    ['afterRme', 'rme', 'left'],
+    ['diff', 'diff', 'center'],
+  ]);
+
+  const beforeRefLink = `${config.repoLink}/commit/${config.beforeSha}`;
+  const afterRefLink = `${config.repoLink}/commit/${config.afterSha}`;
+
+  return `Measured performance changes between [${config.beforeSha}](${beforeRefLink}) and [${config.afterSha}](${afterRefLink}).
+<details>
+<summary>Show table</summary>
+
+${stringTable}
+
+> [!NOTE]
+> **hz** – the number of operations per second (higher – better)
+> **rme** – relative margin of error
+
+</details>
+
+Full log and details of [before](${config.beforeBenchStepLink}) and [after](${config.afterBenchStepLink}) benchmark lunch.`;
+}
+
+const hzFormatter = Intl.NumberFormat('en-US', {
+  maximumFractionDigits: 2,
+});
+
+const rmeFormatter = Intl.NumberFormat('en-US', {
+  maximumFractionDigits: 2,
+  style: 'percent',
+});
+
+const diffFormatter = Intl.NumberFormat('en-US', {
+  maximumFractionDigits: 2,
+  style: 'percent',
+});
+
+/**
+ *
+ * @param {BenchmarkSuiteResult} before
+ * @param {BenchmarkSuiteResult} after
+ */
+export const compare = (before, after) => {
+  const beforeResults = before.testResults;
+  const afterResults = after.testResults;
+
+  const table = [];
+
+  for (const [description, cases] of Object.entries(afterResults)) {
+    cases.sort((a, b) => a.rank - b.rank);
+    for (const [index, afterCase] of cases.entries()) {
+      const beforeCase = beforeResults[description].find(
+        (bench) => bench.name === afterCase.name
+      );
+      const row = {
+        desc: index === 0 ? description : '',
+        name: afterCase.name,
+        afterHz: hzFormatter.format(afterCase.hz),
+        afterRme: '±' + rmeFormatter.format(afterCase.rme / 100),
+      };
+      if (beforeCase) {
+        row.beforeHz = hzFormatter.format(beforeCase.hz);
+        row.beforeRme = '±' + rmeFormatter.format(beforeCase.rme / 100);
+
+        const diffValue = (afterCase.hz - beforeCase.hz) / beforeCase.hz;
+        row.diff = diffFormatter.format(diffValue);
+        if (diffValue < 0) {
+          row.diff = `**${row.diff}**`;
+        }
+      }
+      table.push(row);
+    }
+    // empty row as a separator in table
+    table.push({});
+  }
+  // remove last empty row
+  table.pop();
+  return table;
+};
+
+/**
+ *
+ * @param {object[]} objects
+ * @param {Array[string, string, "left" | "right" | "center"]} fields
+ * @returns
+ */
+export function generateMarkdownTable(objects, columns) {
+  const alignMap = {
+    left: ':---',
+    right: '---:',
+    center: ':---:',
+  };
+  const headers = columns.map(([, header]) => `${header} |`);
+  const aligns = columns.map(([, , align]) => `${alignMap[align]} |`);
+  let table = `| ${headers.join(' ')}\n| ${aligns.join(' ')}\n`;
+  objects.forEach((obj) => {
+    let row = columns.map(([field]) => obj[field]);
+    table += `| ${row.map((value) => `${value ?? ''} |`).join(' ')}\n`;
+  });
+  return table;
+}
+
+/**
+ * @typedef {Object} BenchmarkResult
+ * @property {string} name
+ * @property {number} rank
+ * @property {number} rme
+ * @property {number} totalTime
+ * @property {number} min
+ * @property {number} max
+ * @property {number} hz
+ * @property {number} period
+ * @property {number} mean
+ * @property {number} variance
+ * @property {number} sd
+ * @property {number} sem
+ * @property {number} df
+ * @property {number} critical
+ * @property {number} moe
+ * @property {number} p75
+ * @property {number} p99
+ * @property {number} p995
+ * @property {number} p999
+ */
+
+/**
+ * An object representing the results of a test suite.
+ *
+ * @typedef {Object} BenchmarkSuiteResult
+ * @property {number} numTotalTestSuites
+ * @property {number} numTotalTests
+ * @property {Object.<string, Array<BenchmarkResult>>} testResults
+ */