Integrate hyperlight-ci into CI workflows and Just recipes

- Add cargo alias (`cargo ci`) for convenient hyperlight-ci invocation
- Update dep_benchmarks workflow to use `cargo ci bench` and generate
  a markdown report via `cargo ci bench-report`, posting results as
  a PR comment per hypervisor/cpu matrix entry
- Add benchmarks job to ValidatePullRequest workflow with hypervisor
  and cpu matrix, gated behind docs-only and build-guests checks
- Grant pull-requests: write permission for PR comment posting
- Simplify Justfile bench recipes to delegate to `cargo ci bench`
- Update benchmarking docs to reflect the new workflow

Signed-off-by: Jorge Prendes <jorge.prendes@gmail.com>

Author: jprendes

.cargo/config.toml

@@ -0,0 +1,2 @@
+[alias] # command aliases
+ci = ["run", "--quiet", "--package=hyperlight-ci", "--"]

.github/workflows/ValidatePullRequest.yml

@@ -15,7 +15,7 @@ concurrency:
 
 permissions:
   contents: write
-  pull-requests: read
+  pull-requests: write
 
 jobs:
   docs-pr:
@@ -140,6 +140,28 @@ jobs:
       docs_only: ${{ needs.docs-pr.outputs.docs-only }}
     secrets: inherit
 
+  # Run benchmarks and post results as PR comment
+  benchmarks:
+    needs:
+      - docs-pr
+      - build-guests
+    # Required because update-guest-locks is skipped on non-dependabot PRs,
+    # and a skipped dependency transitively skips all downstream jobs.
+    # See: https://github.com/actions/runner/issues/2205
+    if: ${{ !cancelled() && !failure() }}
+    strategy:
+      fail-fast: false
+      matrix:
+        hypervisor: ['hyperv-ws2025', mshv3, kvm]
+        cpu: [amd, intel]
+    uses: ./.github/workflows/dep_benchmarks.yml
+    secrets: inherit
+    with:
+      docs_only: ${{ needs.docs-pr.outputs.docs-only }}
+      hypervisor: ${{ matrix.hypervisor }}
+      cpu: ${{ matrix.cpu }}
+      pr_number: ${{ github.event.pull_request.number }}
+
   spelling:
     name: spell check with typos
     runs-on: ubuntu-latest
@@ -167,6 +189,7 @@ jobs:
       - build-test
       - run-examples
       - fuzzing
+      - benchmarks
       - spelling
       - license-headers
     if: always()

.github/workflows/dep_benchmarks.yml

@@ -56,13 +56,19 @@ on:
         required: false
         type: number
         default: 5
+      pr_number:
+        description: Pull request number to post benchmark results as a comment. If empty, no comment is posted.
+        required: false
+        type: string
+        default: ""
 
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: full
 
 permissions:
   contents: read
+  pull-requests: write
 
 defaults:
   run:
@@ -133,7 +139,38 @@ jobs:
         continue-on-error: true
 
       - name: Run benchmarks
-        run: just bench-ci main
+        run: just bench-ci
+
+      - name: Create benchmarks report
+        run: cargo ci bench-report > target/criterion/benchmark.md
+
+      - name: Post benchmark results to PR
+        if: ${{ inputs.pr_number != '' }}
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const fs = require('fs');
+            const output = fs.readFileSync('target/criterion/benchmark.md', 'utf8');
+
+            const body = [
+              `## Benchmark Results (${{ inputs.hypervisor }} / ${{ inputs.cpu }})`,
+              '',
+              '<details>',
+              '<summary>Click to expand</summary>',
+              '',
+              '```',
+              output,
+              '```',
+              '',
+              '</details>',
+            ].join('\n');
+
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: Number('${{ inputs.pr_number }}'),
+              body: body,
+            });
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:

Justfile

@@ -177,7 +177,7 @@ run-examples-like-ci config=default-target hypervisor="kvm":
 
 benchmarks-like-ci config=default-target hypervisor="kvm":
     @# Run benchmarks
-    {{ if config == "release" { "just bench-ci main" } else { "" } }}
+    {{ if config == "release" { "just bench-ci" } else { "" } }}
 
 fuzz-like-ci target config=default-target hypervisor="kvm":
     @# Run Fuzzing
@@ -400,13 +400,12 @@ bench-download os hypervisor cpu tag="":
     tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1
 
 # Warning: compares to and then OVERWRITES the given baseline
-bench-ci baseline features="":
-    @# Benchmarks are always run with release builds for meaningful results
-    cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }}
+bench-ci features="":
+    cargo ci bench --no-progress {{ if features == "" {''} else { "--features " + features } }}
 
 bench features="":
     @# Benchmarks are always run with release builds for meaningful results
-    cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose
+    cargo ci bench {{ if features == "" {''} else { "--features " + features } }}
 
 ###############
 ### FUZZING ###

docs/benchmarking-hyperlight.md

@@ -72,6 +72,6 @@ Found 1 outliers among 100 measurements (1.00%)
 
 ## Running benchmarks locally
 
-Use `just bench` to run benchmarks with release builds (the only supported configuration). Comparing local benchmark results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub.
+Use `just bench` to run benchmarks with release builds (the only supported configuration). Comparing local benchmark results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci` to compare to (and overwrite) the previous release benchmarks. The name of the baselines stored in GitHub is `base`.
 
 **Important**: The `just bench` command uses release builds by default to ensure meaningful performance measurements. For profiling purposes, you can compile benchmarks with debug symbols by running `cargo bench` directly.

src/hyperlight_ci/src/bench/discovery.rs

@@ -68,9 +68,18 @@ impl BenchmarkDiscovery {
             if let Some(filenames) = msg.get("filenames").and_then(|f| f.as_array()) {
                 for f in filenames {
                     if let Some(path) = f.as_str() {
-                        if !path.ends_with(".d") {
-                            binaries.push(PathBuf::from(path));
+                        // Skip non-executable artifacts:
+                        //   .d    = dep-info files (all platforms)
+                        //   .pdb  = debug symbols (Windows)
+                        //   .dSYM = debug symbol bundles (macOS)
+                        //   .dwp  = DWARF packages (Linux, split-debuginfo)
+                        //   .lib  = import libraries (Windows)
+                        //   .exp  = export files (Windows)
+                        let dominated = [".d", ".pdb", ".dSYM", ".dwp", ".lib", ".exp"];
+                        if dominated.iter().any(|ext| path.ends_with(ext)) {
+                            continue;
                         }
+                        binaries.push(PathBuf::from(path));
                     }
                 }
             }