Integrate hyperlight-ci into CI workflows and Just recipes

- Add cargo alias (`cargo ci`) for convenient hyperlight-ci invocation
- Update dep_benchmarks workflow to use `cargo ci bench` and generate
  a markdown report via `cargo ci bench-report`, posting results as
  a PR comment per hypervisor/cpu matrix entry
- Add benchmarks job to ValidatePullRequest workflow with hypervisor
  and cpu matrix, gated behind docs-only and build-guests checks
- Grant pull-requests: write permission for PR comment posting
- Simplify Justfile bench recipes to delegate to `cargo ci bench`
- Update benchmarking docs to reflect the new workflow

Signed-off-by: Jorge Prendes <jorge.prendes@gmail.com>

Author: jprendes

.cargo/config.toml

@@ -0,0 +1,2 @@
+[alias] # command aliases
+ci = ["run", "--quiet", "--package=hyperlight-ci", "--"]

.github/workflows/ValidatePullRequest.yml

@@ -15,7 +15,7 @@ concurrency:
 
 permissions:
   contents: write
-  pull-requests: read
+  pull-requests: write
 
 jobs:
   docs-pr:
@@ -140,6 +140,86 @@ jobs:
       docs_only: ${{ needs.docs-pr.outputs.docs-only }}
     secrets: inherit
 
+  # Run benchmarks and post results as PR comment
+  benchmarks:
+    needs:
+      - docs-pr
+      - build-guests
+    # Required because update-guest-locks is skipped on non-dependabot PRs,
+    # and a skipped dependency transitively skips all downstream jobs.
+    # See: https://github.com/actions/runner/issues/2205
+    if: ${{ !cancelled() && !failure() }}
+    strategy:
+      fail-fast: false
+      matrix:
+        hypervisor: ['hyperv-ws2025', mshv3, kvm]
+        cpu: [amd, intel]
+    uses: ./.github/workflows/dep_benchmarks.yml
+    secrets: inherit
+    with:
+      docs_only: ${{ needs.docs-pr.outputs.docs-only }}
+      hypervisor: ${{ matrix.hypervisor }}
+      cpu: ${{ matrix.cpu }}
+
+  # Collect all benchmark reports and post a single combined PR comment
+  benchmark-comment:
+    needs: benchmarks
+    if: ${{ !cancelled() && !failure() }}
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Download all benchmark reports
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          pattern: benchmark-report_*
+          path: reports/
+
+      - name: Post combined benchmark results to PR
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const reportsDir = 'reports';
+            if (!fs.existsSync(reportsDir)) {
+              console.log('No benchmark reports found, skipping comment.');
+              return;
+            }
+
+            // Collect all report files from subdirectories
+            const sections = [];
+            const dirs = fs.readdirSync(reportsDir).sort();
+            for (const dir of dirs) {
+              const mdPath = path.join(reportsDir, dir, 'benchmark.md');
+              if (!fs.existsSync(mdPath)) continue;
+
+              // Extract hypervisor/cpu from artifact name: benchmark-report_OS_hypervisor_cpu
+              const parts = dir.replace('benchmark-report_', '').split('_');
+              const os = parts[0];
+              const hypervisor = parts.slice(1, -1).join('_');
+              const cpu = parts[parts.length - 1];
+              const label = `${hypervisor} / ${cpu} (${os})`;
+
+              const content = fs.readFileSync(mdPath, 'utf8').trim();
+              sections.push(`<details>\n<summary><b>${label}</b></summary>\n\n${content}\n\n</details>`);
+            }
+
+            if (sections.length === 0) {
+              console.log('No benchmark report content found, skipping comment.');
+              return;
+            }
+
+            const body = `## Benchmark Results\n\n${sections.join('\n\n')}`;
+
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: body,
+            });
+
   spelling:
     name: spell check with typos
     runs-on: ubuntu-latest
@@ -167,6 +247,8 @@ jobs:
       - build-test
       - run-examples
       - fuzzing
+      - benchmarks
+      - benchmark-comment
       - spelling
       - license-headers
     if: always()

.github/workflows/dep_benchmarks.yml

@@ -56,7 +56,6 @@ on:
         required: false
         type: number
         default: 5
-
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: full
@@ -133,7 +132,17 @@ jobs:
         continue-on-error: true
 
       - name: Run benchmarks
-        run: just bench-ci main
+        run: just bench-ci
+
+      - name: Create benchmarks report
+        run: cargo ci bench-report > target/criterion/benchmark.md
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: benchmark-report_${{ runner.os }}_${{ inputs.hypervisor }}_${{ inputs.cpu }}
+          path: target/criterion/benchmark.md
+          if-no-files-found: error
+          retention-days: 1
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
         with:

Justfile

@@ -177,7 +177,7 @@ run-examples-like-ci config=default-target hypervisor="kvm":
 
 benchmarks-like-ci config=default-target hypervisor="kvm":
     @# Run benchmarks
-    {{ if config == "release" { "just bench-ci main" } else { "" } }}
+    {{ if config == "release" { "just bench-ci" } else { "" } }}
 
 fuzz-like-ci target config=default-target hypervisor="kvm":
     @# Run Fuzzing
@@ -400,13 +400,12 @@ bench-download os hypervisor cpu tag="":
     tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1
 
 # Warning: compares to and then OVERWRITES the given baseline
-bench-ci baseline features="":
-    @# Benchmarks are always run with release builds for meaningful results
-    cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }}
+bench-ci features="":
+    cargo ci bench --no-progress {{ if features == "" {''} else { "--features " + features } }}
 
 bench features="":
     @# Benchmarks are always run with release builds for meaningful results
-    cargo bench --profile=release {{ if features =="" {''} else { "--features " + features } }} -- --verbose
+    cargo ci bench {{ if features == "" {''} else { "--features " + features } }}
 
 ###############
 ### FUZZING ###

docs/benchmarking-hyperlight.md

@@ -72,6 +72,6 @@ Found 1 outliers among 100 measurements (1.00%)
 
 ## Running benchmarks locally
 
-Use `just bench` to run benchmarks with release builds (the only supported configuration). Comparing local benchmark results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci main` to compare to (and overwrite) the previous release benchmarks. Note that `main` is the name of the baselines stored in GitHub.
+Use `just bench` to run benchmarks with release builds (the only supported configuration). Comparing local benchmark results to github-saved benchmarks doesn't make much sense, since you'd be using different hardware, but you can use `just bench-download os hypervisor [tag] ` to download and extract the GitHub release benchmarks to the correct place folder. You can then run `just bench-ci` to compare to (and overwrite) the previous release benchmarks. The name of the baselines stored in GitHub is `base`.
 
 **Important**: The `just bench` command uses release builds by default to ensure meaningful performance measurements. For profiling purposes, you can compile benchmarks with debug symbols by running `cargo bench` directly.

src/hyperlight_ci/src/bench/discovery.rs

@@ -68,9 +68,18 @@ impl BenchmarkDiscovery {
             if let Some(filenames) = msg.get("filenames").and_then(|f| f.as_array()) {
                 for f in filenames {
                     if let Some(path) = f.as_str() {
-                        if !path.ends_with(".d") {
-                            binaries.push(PathBuf::from(path));
+                        // Skip non-executable artifacts:
+                        //   .d    = dep-info files (all platforms)
+                        //   .pdb  = debug symbols (Windows)
+                        //   .dSYM = debug symbol bundles (macOS)
+                        //   .dwp  = DWARF packages (Linux, split-debuginfo)
+                        //   .lib  = import libraries (Windows)
+                        //   .exp  = export files (Windows)
+                        let dominated = [".d", ".pdb", ".dSYM", ".dwp", ".lib", ".exp"];
+                        if dominated.iter().any(|ext| path.ends_with(ext)) {
+                            continue;
                         }
+                        binaries.push(PathBuf::from(path));
                     }
                 }
             }