benchmark/.github/workflows/bench.yml at main · twilic/benchmark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
name: Benchmark

on:
  workflow_dispatch:
    inputs:
      backend:
        description: "N-API or WASM"
        type: choice
        options:
          - napi
          - wasm
        default: napi
      mode:
        description: "Benchmark suite"
        type: choice
        options:
          - full
          - max
        default: full
      twilic_vs_msgpack_only:
        description: "Omit JSON baseline rows and tasks"
        type: boolean
        default: false
      time_ms:
        description: "Time per task (ms)"
        type: string
        default: "1000"
      warmup_ms:
        description: "Warmup per task (ms)"
        type: string
        default: "250"

concurrency:
  group: bench-${{ github.repository }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  bench:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - name: Checkout benchmark
        uses: actions/checkout@v4
        with:
          path: workspace/benchmark
          persist-credentials: false

      - name: Checkout twilic-js
        uses: actions/checkout@v4
        with:
          repository: ${{ github.repository_owner }}/twilic-js
          path: workspace/twilic-js
          persist-credentials: false

      - name: Checkout twilic-rust
        uses: actions/checkout@v4
        with:
          repository: ${{ github.repository_owner }}/twilic-rust
          path: workspace/twilic-rust
          persist-credentials: false

      - name: Setup pnpm
        uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
        with:
          version: 10.18.1

      - name: Setup Node.js
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
        with:
          node-version: 24
          cache: pnpm
          cache-dependency-path: |
            workspace/twilic-js/pnpm-lock.yaml
            workspace/benchmark/pnpm-lock.yaml

      - name: Setup Rust
        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable

      - name: Install wasm-pack
        uses: jetli/wasm-pack-action@0d096b08b4e5a7de8c28de67e11e945404e9eefa # v0.4.0

      - name: Build twilic-js
        working-directory: workspace/twilic-js
        run: |
          pnpm install --frozen-lockfile
          pnpm build

      - name: Install bench dependencies
        working-directory: workspace/benchmark
        run: pnpm install --frozen-lockfile

      - name: Run benchmark
        working-directory: workspace/benchmark
        env:
          BACKEND: ${{ inputs.backend }}
          MODE: ${{ inputs.mode }}
          TIME_MS: ${{ inputs.time_ms }}
          WARMUP_MS: ${{ inputs.warmup_ms }}
          TWILIC_VS_MSGPACK_ONLY: ${{ inputs.twilic_vs_msgpack_only }}
        run: |
          set -euo pipefail
          for var in TIME_MS WARMUP_MS; do
            value="${!var}"
            if ! [[ "$value" =~ ^[0-9]+$ ]]; then
              echo "$var must be a positive integer, got: $value" >&2
              exit 1
            fi
          done
          extra=()
          if [ "$TWILIC_VS_MSGPACK_ONLY" = "true" ]; then
            extra+=(--twilic-vs-msgpack-only)
          fi
          pnpm exec tsx src/benchmark.ts \
            --markdown-out "${GITHUB_STEP_SUMMARY}" \
            --backend "$BACKEND" \
            --mode "$MODE" \
            --time-ms "$TIME_MS" \
            --warmup-ms "$WARMUP_MS" \
            "${extra[@]}"