feat(benchmarks): Added intital attempt to generate basic summary via LLM

Author: PyDataBlog

.github/workflows/CI.yml

@@ -56,12 +56,14 @@ jobs:
           version: "1.11"
 
       - name: Run and Compare Benchmarks
+        env:
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
         run: |
           # Install python dependencies
           pip install uv
           uv venv
           source .venv/bin/activate
-          uv pip install simstring-fast pandas maturin tabulate
+          uv pip install simstring-fast pandas maturin tabulate pydantic-ai tenacity
           rm -rf target/wheels
           maturin build --release
           uv pip install target/wheels/*.whl

benches/compare_benches.py

@@ -3,6 +3,26 @@
 from pathlib import Path
 
 import pandas as pd
+from pydantic_ai import Agent
+from tenacity import retry, stop_after_attempt, wait_exponential
+
+agent = Agent(
+    "google-gla:gemini-2.5-pro",
+    system_prompt=(
+        "Directly generate a markdown summary of the provided benchmark results. "
+        "Do not include any conversational filler or introductory sentences. "
+        "Your response should begin immediately with the summary content, "
+        "structured with markdown sub-headings (e.g., '#### Summary', '##### Insert Performance', '#### Search Performance'). "
+        "The summary should analyze insert and search performance, "
+        "compare the different implementations, and include speedup metrics."
+    ),
+)
+
+
+@retry(wait=wait_exponential(multiplier=1, min=4, max=10), stop=stop_after_attempt(3))
+def generate_summary(agent: Agent, benchmark_markdown_output: str) -> str:
+    results = agent.run_sync(benchmark_markdown_output)
+    return results.output
 
 
 def compare_benchmarks():
@@ -62,6 +82,10 @@ def compare_benchmarks():
                 markdown_output = group.to_markdown(index=False)
                 if markdown_output:
                     f.write(markdown_output)
+                    # Generate a comparision summary section via LLM which includes the speedup gains and/os losses
+                    f.write("\n\n#### AI Summary\n")
+                    summary = generate_summary(agent, markdown_output)
+                    f.write(summary)
                 f.write("\n\n")
 
         print("--- Finished benchmark comparison successfully ---")

benches/run_benches.py

@@ -63,4 +63,3 @@ def run_benchmarks():
 if __name__ == "__main__":
     run_benchmarks()
     compare_benchmarks()
-