feat(bindings): Added strings method for getting all stored strings & more search improvements

Author: PyDataBlog

src/database/hashdb.rs

@@ -54,11 +54,7 @@ impl HashDb {
 impl Database for HashDb {
     fn insert(&mut self, text: String) {
         let mut interner = self.interner.lock().unwrap();
-        let mut features = self.feature_extractor.features(&text, &mut interner);
-        // FIX: Hmm append_feature_counts does counts. Should this sorting be moved there and make
-        // dedup redundant?
-        features.sort_unstable();
-        features.dedup();
+        let features = self.feature_extractor.features(&text, &mut interner);
         let size = features.len();
         let string_id = self.strings.len();
 
@@ -102,4 +98,8 @@ impl Database for HashDb {
     fn interner(&self) -> Arc<Mutex<Rodeo>> {
         Arc::clone(&self.interner)
     }
+
+    fn total_strings(&self) -> usize {
+        self.strings.len()
+    }
 }

src/database/mod.rs

@@ -1,8 +1,8 @@
 mod hashdb;
 
 use crate::extractors::FeatureExtractor;
-use rustc_hash::FxHashSet;
 use lasso::{Rodeo, Spur};
+use rustc_hash::FxHashSet;
 use std::sync::{Arc, Mutex};
 
 pub type StringId = usize;
@@ -16,6 +16,7 @@ pub trait Database: Send + Sync {
     fn feature_extractor(&self) -> &dyn FeatureExtractor;
     fn max_feature_len(&self) -> usize;
     fn interner(&self) -> Arc<Mutex<Rodeo>>;
+    fn total_strings(&self) -> usize;
 }
 
 pub use hashdb::HashDb;

src/extractors/character_ngrams.rs

@@ -28,21 +28,34 @@ impl FeatureExtractor for CharacterNgrams {
             return vec![];
         }
 
-        let mut ngrams = Vec::new();
-        let padding = self.endmarker.repeat(self.n.saturating_sub(1));
+        // Pre-calculate capacity to avoid reallocations
+        let text_len = text.chars().count();
+        let padding_len = self.n.saturating_sub(1);
+        let total_len = text_len + 2 * padding_len;
 
-        // Create an iterator that includes padding
-        let padded_text_iter = padding.chars().chain(text.chars()).chain(padding.chars());
+        if total_len < self.n {
+            return vec![];
+        }
+
+        let expected_ngrams = total_len - self.n + 1;
+        let mut ngrams = Vec::with_capacity(expected_ngrams);
+
+        let padding = self.endmarker.repeat(padding_len);
 
-        // Use a buffer to collect characters for each n-gram
-        let mut buffer: Vec<char> = Vec::with_capacity(self.n);
+        // collect chars once, then slice
+        let mut all_chars = Vec::with_capacity(total_len);
+        all_chars.extend(padding.chars());
+        all_chars.extend(text.chars());
+        all_chars.extend(padding.chars());
 
-        for ch in padded_text_iter {
-            buffer.push(ch);
-            if buffer.len() == self.n {
-                ngrams.push(buffer.iter().collect::<String>());
-                buffer.remove(0);
+        // Generate n-grams using efficient windowing
+        for window in all_chars.windows(self.n) {
+            // Pre-allocate string with known capacity
+            let mut ngram = String::with_capacity(self.n * 4); // Assume max 4 bytes per char
+            for &ch in window {
+                ngram.push(ch);
             }
+            ngrams.push(ngram);
         }
 
         super::append_feature_counts(interner, ngrams)

src/extractors/mod.rs

@@ -1,20 +1,28 @@
 mod character_ngrams;
 mod word_ngrams;
 
-use rustc_hash::FxHashMap;
 use lasso::{Rodeo, Spur};
+use rustc_hash::FxHashMap;
+use std::fmt::Write;
 
 /// Takes a list of features and makes each one unique by appending its occurrence count,
-/// then interns the result.
+/// then interns the result and returns them sorted.
 fn append_feature_counts(interner: &mut Rodeo, features: Vec<String>) -> Vec<Spur> {
     let mut counter: FxHashMap<String, usize> = FxHashMap::default();
     let mut unique_features = Vec::with_capacity(features.len());
+
     for val in features {
         let count = counter.entry(val.clone()).or_insert(0);
         *count += 1;
-        let unique_string = format!("{}{}", val, *count);
+
+        let mut unique_string = String::with_capacity(val.len() + 8); // Extra space for count
+        unique_string.push_str(&val);
+        write!(&mut unique_string, "{count}",).unwrap();
+
         unique_features.push(interner.get_or_intern(unique_string));
     }
+
+    unique_features.sort_unstable();
     unique_features
 }
 

src/extractors/word_ngrams.rs

@@ -32,7 +32,7 @@ impl FeatureExtractor for WordNgrams {
 
         let tokens = text.split(&self.splitter).filter(|s| !s.is_empty());
 
-        // Create an iterator that includes padding
+        // an iterator that includes padding
         let padded_tokens_iter = std::iter::once(self.padder.as_str())
             .chain(tokens)
             .chain(std::iter::once(self.padder.as_str()));

src/measures/cosine.rs

@@ -1,6 +1,5 @@
-use super::Measure;
+use super::{compute_intersection_size, Measure};
 use crate::database::Database;
-
 use lasso::Spur;
 
 #[derive(Default, Clone, Copy)]
@@ -28,25 +27,10 @@ impl Measure for Cosine {
             return 0.0;
         }
 
-        let mut intersection_size = 0;
-        let mut i = 0;
-        let mut j = 0;
-
-        while i < x.len() && j < y.len() {
-            if x[i] == y[j] {
-                intersection_size += 1;
-                i += 1;
-                j += 1;
-            } else if x[i] < y[j] {
-                i += 1;
-            } else {
-                j += 1;
-            }
-        }
-
+        let intersection_size = compute_intersection_size(x, y);
         let denominator = (x.len() as f64 * y.len() as f64).sqrt();
 
-        if denominator == 0.0 {
+        if denominator == 0.0 || !denominator.is_finite() {
             0.0
         } else {
             intersection_size as f64 / denominator

src/measures/dice.rs

@@ -1,6 +1,5 @@
-use super::Measure;
+use super::{compute_intersection_size, Measure};
 use crate::database::Database;
-
 use lasso::Spur;
 
 #[derive(Default, Clone, Copy)]
@@ -34,22 +33,7 @@ impl Measure for Dice {
             return 0.0;
         }
 
-        let mut intersection_size = 0;
-        let mut i = 0;
-        let mut j = 0;
-
-        while i < x.len() && j < y.len() {
-            if x[i] == y[j] {
-                intersection_size += 1;
-                i += 1;
-                j += 1;
-            } else if x[i] < y[j] {
-                i += 1;
-            } else {
-                j += 1;
-            }
-        }
-
+        let intersection_size = compute_intersection_size(x, y);
         let denominator = (x.len() + y.len()) as f64;
 
         if denominator == 0.0 {

src/measures/exact_match.rs

@@ -1,6 +1,5 @@
 use super::Measure;
 use crate::database::Database;
-
 use lasso::Spur;
 
 #[derive(Default, Clone, Copy)]
@@ -36,4 +35,3 @@ impl Measure for ExactMatch {
         }
     }
 }
-

src/measures/jaccard.rs

@@ -1,6 +1,5 @@
-use super::Measure;
+use super::{compute_intersection_size, Measure};
 use crate::database::Database;
-
 use lasso::Spur;
 
 #[derive(Default, Clone, Copy)]
@@ -30,22 +29,7 @@ impl Measure for Jaccard {
             return 0.0;
         }
 
-        let mut intersection_size = 0;
-        let mut i = 0;
-        let mut j = 0;
-
-        while i < x.len() && j < y.len() {
-            if x[i] == y[j] {
-                intersection_size += 1;
-                i += 1;
-                j += 1;
-            } else if x[i] < y[j] {
-                i += 1;
-            } else {
-                j += 1;
-            }
-        }
-
+        let intersection_size = compute_intersection_size(x, y);
         let union_size = (x.len() + y.len() - intersection_size) as f64;
 
         if union_size == 0.0 {

src/measures/mod.rs

@@ -15,26 +15,26 @@ pub trait Measure: Send + Sync {
     fn similarity(&self, x: &[Spur], y: &[Spur]) -> f64;
 }
 
-// FIX: All measures use the same intersection logic so this function can clean up measures module:
-//fn compute_intersection_size(x: &[Spur], y: &[Spur]) -> usize {
-//    let mut intersection_size = 0;
-//    let mut i = 0;
-//    let mut j = 0;
-//
-//    while i < x.len() && j < y.len() {
-//        match x[i].cmp(&y[j]) {
-//            std::cmp::Ordering::Equal => {
-//                intersection_size += 1;
-//                i += 1;
-//                j += 1;
-//            }
-//            std::cmp::Ordering::Less => i += 1,
-//            std::cmp::Ordering::Greater => j += 1,
-//        }
-//    }
-//
-//    intersection_size
-//}
+// Helper function which computes the number of intersections between to vec of Spur
+pub(crate) fn compute_intersection_size(x: &[Spur], y: &[Spur]) -> usize {
+    let mut intersection_size = 0;
+    let mut i = 0;
+    let mut j = 0;
+
+    while i < x.len() && j < y.len() {
+        match x[i].cmp(&y[j]) {
+            std::cmp::Ordering::Equal => {
+                intersection_size += 1;
+                i += 1;
+                j += 1;
+            }
+            std::cmp::Ordering::Less => i += 1,
+            std::cmp::Ordering::Greater => j += 1,
+        }
+    }
+
+    intersection_size
+}
 pub use cosine::Cosine;
 pub use dice::Dice;
 pub use exact_match::ExactMatch;