feat(ci): Initial benchmark support for original C++ implementation (#54)

* feat(ci): Initial benchmark support for original C++ implementation (including original Python bindings)

* chore(benchmarks): switched to generic searches like original implementation does

* chore(ci): updated github actions versions

Author: PyDataBlog

.github/workflows/CI.yml

@@ -20,7 +20,7 @@ jobs:
       contents: write
       pull-requests: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           # On pull requests, check out the head ref so we can commit to it
           ref: ${{ github.head_ref }}
@@ -32,7 +32,7 @@ jobs:
           sudo apt-get install -y software-properties-common
           sudo add-apt-repository universe
           sudo apt-get update
-          sudo apt-get install -y build-essential simstring-bin
+          sudo apt-get install -y build-essential autoconf automake libtool swig python3-dev simstring-bin
 
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
@@ -41,19 +41,19 @@ jobs:
           components: rustfmt, clippy
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.13"
+          python-version: "3.14"
 
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: "3.3"
+          ruby-version: "3.4"
 
       - name: Set up Julia
         uses: julia-actions/setup-julia@v2
         with:
-          version: "1.11"
+          version: "1"
 
       - name: Cache python dependencies
         uses: actions/cache@v4
@@ -97,7 +97,7 @@ jobs:
 
       - name: Post benchmark results to PR
         if: github.event_name == 'pull_request'
-        uses: actions/github-script@v7
+        uses: actions/github-script@v8
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -157,7 +157,7 @@ jobs:
             target: aarch64-apple-darwin
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
@@ -183,9 +183,9 @@ jobs:
           restore-keys: ${{ runner.os }}-${{ matrix.target }}-cargo-
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.13"
+          python-version: "3.14t"
 
       - name: Cache python dependencies
         uses: actions/cache@v4
@@ -250,8 +250,8 @@ jobs:
       matrix:
         os: [windows-latest, macos-latest]
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.10"
 
@@ -273,8 +273,8 @@ jobs:
     name: Build manylinux wheels
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.10"
 
@@ -305,8 +305,8 @@ jobs:
           - os: macos-14
             python-version: "3.10"
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -360,8 +360,8 @@ jobs:
     permissions:
       id-token: write
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
+      - uses: actions/checkout@v5
+      - uses: actions/download-artifact@v6
         with:
           pattern: wheels-*
           path: dist
@@ -414,7 +414,7 @@ jobs:
     needs: test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
@@ -423,9 +423,9 @@ jobs:
         run: cargo run --example basic --release
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.13"
+          python-version: "3.14"
 
       - name: Cache python dependencies
         uses: actions/cache@v4
@@ -459,7 +459,7 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
@@ -481,7 +481,7 @@ jobs:
       contents: write
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0 # get all tags
 

.github/workflows/coverage.yml

@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Install rust
         uses: dtolnay/rust-toolchain@stable
@@ -27,7 +27,7 @@ jobs:
           sudo apt-get install -y software-properties-common python3-dev
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.14"
 

.gitignore

@@ -28,6 +28,9 @@ Session.vim
 .vscode/
 .DS_Store
 bench
+.simstring_cpp
+.simstring_cpp/
+benches/.simstring_cpp/
 .venv
 uv.lock
 tests/python/__pycache__

README.md

@@ -126,6 +126,15 @@ License
 
 This project is licensed under the MIT License.
 
+## Benchmarks
+
+The `benches/run_benches.py` harness compares several language bindings (Rust, Python, Julia, Ruby, C++).
+
+- `git`, `autoconf`, `automake`, `libtool`, `make`, `python`, `uv` and a C++ compiler (`g++`) to build the C++ CLI.
+
+The C++ sources are cloned into `benches/.simstring_cpp/` and a local copy of the `simstring` binary is installed
+under that directory. If you need to rebuild from scratch, remove `benches/.simstring_cpp/` before re-running the benchmark suite.
+
 ## Acknowledgements
 
 Inspired by the [SimString](https://www.chokkan.org/software/simstring/) project.

benches/bench.jl

@@ -55,7 +55,7 @@ function bench_search(results::Vector)
         for threshold in [0.6, 0.7, 0.8]
             b = @benchmarkable begin
                 for term in $search_terms
-                    SimString.search(SimString.Cosine(), $db, term; α=$threshold, ranked=true)
+                    SimString.search(SimString.Cosine(), $db, term; α=$threshold, ranked=false)
                 end
             end samples=100 seconds=20
 

benches/bench.py

@@ -98,7 +98,7 @@ def bench_search(results: list):
 
                 def benchmark_iteration():
                     for term in search_terms:
-                        searcher.ranked_search(term, threshold)
+                        searcher.search(term, threshold)
 
                 duration = measure_time(benchmark_iteration)
                 measurements.append(duration)

benches/bench.rb

@@ -62,7 +62,7 @@ def bench_search(results)
       while Time.now - start_time < 20 && iterations < 100
         time = Benchmark.realtime do
           search_terms.each do |term|
-            matcher.ranked_search(term, threshold)
+            matcher.search(term, threshold)
           end
         end
         measurements << time

benches/bench.rs

@@ -48,16 +48,14 @@ fn bench_insert(results: &mut Vec<BenchmarkResult>) {
         let mut iteration = 0;
 
         while start_time.elapsed() < Duration::from_secs(20) && iteration < 100 {
-            let setup_start = Instant::now();
             let fe = Arc::new(CharacterNgrams::new(*ngram_size, " "));
             let mut db = HashDb::new(fe);
-            let setup_duration = setup_start.elapsed();
 
             let start = Instant::now();
             for company in &companies {
                 db.insert(company.clone());
             }
-            let duration = start.elapsed() - setup_duration;
+            let duration = start.elapsed();
             measurements.push(duration.as_secs_f64() * 1000.0);
             iteration += 1;
         }
@@ -114,7 +112,7 @@ fn bench_search(results: &mut Vec<BenchmarkResult>) {
             while start_time.elapsed() < Duration::from_secs(20) && iteration < 100 {
                 let start = Instant::now();
                 for term in &search_terms {
-                    let _ = searcher.ranked_search(term, *threshold).unwrap();
+                    let _ = searcher.search(term, *threshold).unwrap();
                 }
                 let duration = start.elapsed();
                 measurements.push(duration.as_secs_f64() * 1000.0);

benches/bench_cpp.sh

@@ -93,7 +93,7 @@ def bench_search(results: list):
 
                 def benchmark_iteration():
                     for term in search_terms:
-                        searcher.ranked_search(term, threshold)
+                        searcher.search(term, threshold)
 
                 duration = measure_time(benchmark_iteration)
                 measurements.append(duration)