Repository navigation
Atomesh Mocker Benchmark #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Atomesh Mocker Benchmark | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - 'atom/mesh/**' | |
| - '.github/scripts/atomesh/atomesh_mocker_benchmark.sh' | |
| - '.github/scripts/atomesh/atomesh_mocker_benchmark_summary.py' | |
| - '.github/workflows/atomesh-mocker-benchmark.yaml' | |
| - '.github/dashboard/atomesh_mocker_index.html' | |
| - 'docs/assets/atomesh_logo.png' | |
| pull_request: | |
| branches: [main] | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| paths: | |
| - 'atom/mesh/**' | |
| - '.github/scripts/atomesh/atomesh_mocker_benchmark.sh' | |
| - '.github/scripts/atomesh/atomesh_mocker_benchmark_summary.py' | |
| - '.github/workflows/atomesh-mocker-benchmark.yaml' | |
| - '.github/dashboard/atomesh_mocker_index.html' | |
| - 'docs/assets/atomesh_logo.png' | |
| schedule: | |
| # Nightly at 02:00 Beijing time (18:00 UTC) | |
| - cron: '0 18 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| suite: | |
| description: 'Benchmark suite: smoke runs 1P1D/c=1 sanity check; full runs 1P1D, 2P1D, 3P1D across c=1,2,4,8,16' | |
| required: false | |
| default: 'full' | |
| type: choice | |
| options: | |
| - smoke | |
| - full | |
| publish_dashboard: | |
| description: 'Publish workflow_dispatch results to the benchmark dashboard' | |
| required: false | |
| default: false | |
| type: boolean | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| permissions: | |
| actions: read | |
| contents: write | |
| jobs: | |
| run_atomesh_test_harness: | |
| if: ${{ !github.event.pull_request || github.event.pull_request.draft == false }} | |
| name: run_atomesh_test_harness | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout ATOM repo | |
| uses: actions/checkout@v6 | |
| - name: Set up build environment | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y protobuf-compiler | |
| - name: Cache cargo build output | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| atom/mesh/mocker/target | |
| key: atomesh-mocker-cargo-${{ runner.os }}-${{ hashFiles('atom/mesh/Cargo.lock', 'atom/mesh/Cargo.toml', 'atom/mesh/mocker/Cargo.toml') }} | |
| restore-keys: | | |
| atomesh-mocker-cargo-${{ runner.os }}- | |
| - name: Run Atomesh test harness | |
| run: | | |
| set -euo pipefail | |
| cargo test \ | |
| --manifest-path atom/mesh/mocker/Cargo.toml \ | |
| --target-dir atom/mesh/mocker/target/mocker \ | |
| --release \ | |
| test_atomesh_harness | |
| run_atomesh_mocker_benchmark: | |
| if: ${{ !github.event.pull_request || github.event.pull_request.draft == false }} | |
| name: run_atomesh_mocker_benchmark | |
| needs: [run_atomesh_test_harness] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 75 | |
| steps: | |
| - name: Checkout ATOM repo | |
| uses: actions/checkout@v6 | |
| - name: Build benchmark matrix | |
| env: | |
| SUITE: ${{ github.event_name == 'workflow_dispatch' && inputs.suite || 'full' }} | |
| run: | | |
| python3 <<'PY' | |
| import json | |
| import os | |
| suite = os.environ.get("SUITE", "full") | |
| if suite == "smoke": | |
| duration = "30s" | |
| consumer_threads = [1] | |
| topologies = [(1, 1)] | |
| elif suite == "full": | |
| duration = "3m" | |
| consumer_threads = [1, 2, 4, 8, 16] | |
| topologies = [(1, 1), (2, 1), (3, 1)] | |
| else: | |
| raise SystemExit(f"Unsupported suite={suite}") | |
| cells = [] | |
| def add_pd(duration, prefill, decode, consumers): | |
| cells.append({ | |
| "id": f"pd-chat-{prefill}p{decode}d-conc{consumers}", | |
| "display": f"pd-chat {prefill}P{decode}D CONC{consumers}", | |
| "scenario": "pd-chat", | |
| "duration": duration, | |
| "prefill_workers": prefill, | |
| "decode_workers": decode, | |
| "producer_threads": 1, | |
| "consumer_threads": consumers, | |
| }) | |
| for prefill, decode in topologies: | |
| for consumers in consumer_threads: | |
| add_pd(duration, prefill, decode, consumers) | |
| cells_json = json.dumps(cells) | |
| with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env: | |
| env.write(f"CELLS_JSON={cells_json}\n") | |
| print(f"Generated {len(cells)} benchmark cells for suite={suite}") | |
| for cell in cells: | |
| print( | |
| f" {cell['id']}: scenario={cell['scenario']} duration={cell['duration']} " | |
| f"P/D={cell['prefill_workers']}/{cell['decode_workers']} " | |
| f"producer/consumer={cell['producer_threads']}/{cell['consumer_threads']}" | |
| ) | |
| PY | |
| - name: Cache Rust build artifacts | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| atom/mesh/mocker/target | |
| key: atomesh-mocker-cargo-${{ runner.os }}-${{ hashFiles('atom/mesh/Cargo.lock', 'atom/mesh/Cargo.toml', 'atom/mesh/mocker/Cargo.toml') }} | |
| restore-keys: | | |
| atomesh-mocker-cargo-${{ runner.os }}- | |
| - name: Build Atomesh | |
| run: | | |
| set -euo pipefail | |
| sudo apt-get update | |
| sudo apt-get install -y protobuf-compiler | |
| cargo build \ | |
| --manifest-path atom/mesh/mocker/Cargo.toml \ | |
| --target-dir atom/mesh/mocker/target/mocker \ | |
| --release | |
| cargo build \ | |
| --manifest-path atom/mesh/Cargo.toml \ | |
| --target-dir atom/mesh/mocker/target/mesh \ | |
| --release | |
| - name: Run mocker benchmark | |
| env: | |
| RESULT_DIR: atomesh-mocker-results | |
| run: | | |
| set -euo pipefail | |
| chmod +x .github/scripts/atomesh/atomesh_mocker_benchmark.sh | |
| python3 .github/scripts/atomesh/atomesh_mocker_benchmark_summary.py | |
| - name: Dump mocker benchmark-request log | |
| if: always() | |
| run: | | |
| set -euo pipefail | |
| shopt -s nullglob | |
| logs=(atomesh-mocker-results/logs/*/benchmark-request.log) | |
| if [ "${#logs[@]}" -eq 0 ]; then | |
| echo "No Atomesh mocker benchmark-request logs found." | |
| exit 0 | |
| fi | |
| for log in "${logs[@]}"; do | |
| cell="$(basename "$(dirname "$log")")" | |
| echo "::group::benchmark-request ${cell}" | |
| cat "$log" | |
| echo "::endgroup::" | |
| done | |
| - name: Summarize mocker benchmark result | |
| if: always() | |
| run: | | |
| set -euo pipefail | |
| if [ -f "atomesh-mocker-results/benchmark-summary.md" ]; then | |
| cat "atomesh-mocker-results/benchmark-summary.md" | |
| cat "atomesh-mocker-results/benchmark-summary.md" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "No Atomesh mocker benchmark summary was generated." >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Upload benchmark result | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: atomesh-mocker-benchmark-results | |
| path: | | |
| atomesh-mocker-results/*.json | |
| atomesh-mocker-results/*.md | |
| atomesh-mocker-results/logs/ | |
| if-no-files-found: ignore | |
| dashboard: | |
| concurrency: | |
| group: gh-pages-deploy | |
| cancel-in-progress: false | |
| name: Update Mocker Benchmark Dashboard | |
| needs: [run_atomesh_mocker_benchmark] | |
| if: >- | |
| !cancelled() | |
| && needs.run_atomesh_mocker_benchmark.result == 'success' | |
| && ( | |
| github.event_name == 'schedule' | |
| || github.event_name == 'push' | |
| || (github.event_name == 'workflow_dispatch' && inputs.publish_dashboard) | |
| ) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout ATOM repo | |
| uses: actions/checkout@v6 | |
| - name: Download benchmark artifacts | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: atomesh-mocker-benchmark-* | |
| merge-multiple: true | |
| path: atomesh-mocker-results | |
| - name: Build benchmark-action input | |
| run: | | |
| set -euo pipefail | |
| python3 - <<'PY' | |
| import json | |
| from pathlib import Path | |
| entries = [] | |
| for path in sorted(Path("atomesh-mocker-results").glob("*-benchmark-action.json")): | |
| entries.extend(json.loads(path.read_text(encoding="utf-8"))) | |
| Path("atomesh-mocker-dashboard-input.json").write_text( | |
| json.dumps(entries, indent=2), | |
| encoding="utf-8", | |
| ) | |
| print(f"Generated {len(entries)} dashboard entries") | |
| PY | |
| - name: Prepare existing dashboard data | |
| if: hashFiles('atomesh-mocker-dashboard-input.json') != '' | |
| id: prepare_dashboard_data | |
| uses: ./.github/actions/prepare-dashboard-data | |
| with: | |
| data-path: atomesh-mocker-dashboard/data.js | |
| - name: Store benchmark result to dashboard | |
| if: hashFiles('atomesh-mocker-dashboard-input.json') != '' | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| tool: customBiggerIsBetter | |
| output-file-path: atomesh-mocker-dashboard-input.json | |
| gh-pages-branch: gh-pages | |
| benchmark-data-dir-path: atomesh-mocker-dashboard | |
| auto-push: false | |
| max-items-in-chart: 300 | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Validate dashboard data after write | |
| if: hashFiles('atomesh-mocker-dashboard-input.json') != '' | |
| uses: ./.github/actions/validate-dashboard-data | |
| with: | |
| data-path: atomesh-mocker-dashboard/data.js | |
| min-entries: ${{ steps.prepare_dashboard_data.outputs.entry_count }} | |
| - name: Deploy mocker benchmark dashboard to gh-pages | |
| if: hashFiles('atomesh-mocker-dashboard-input.json') != '' | |
| run: | | |
| set -euo pipefail | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| DASHBOARD_TEMPLATE=$(mktemp) | |
| LOGO_ASSET=$(mktemp) | |
| cp .github/dashboard/atomesh_mocker_index.html "$DASHBOARD_TEMPLATE" | |
| cp docs/assets/atomesh_logo.png "$LOGO_ASSET" | |
| CURRENT_SHA=$(git rev-parse HEAD) | |
| git fetch origin gh-pages | |
| git checkout gh-pages | |
| mkdir -p atomesh-mocker-dashboard | |
| cp "$DASHBOARD_TEMPLATE" atomesh-mocker-dashboard/index.html | |
| cp "$LOGO_ASSET" atomesh-mocker-dashboard/atomesh_logo.png | |
| git add atomesh-mocker-dashboard/ | |
| git diff --cached --quiet || git commit -m "Update Atomesh mocker benchmark dashboard" | |
| git push origin gh-pages | |
| git checkout "$CURRENT_SHA" |