Query unigrams and bigrams in query

src/lib.rs

@@ -113,6 +113,16 @@ impl Ranker {
         }
     }
 
+    pub fn get_query_terms(&self) -> JsValue {
+        let tokens = self.query.split_whitespace().collect::<Vec<&str>>();
+        let bigrams = tokens.windows(2).map(|pair| pair.join(" ")).collect::<Vec<String>>();
+        let unique_tokens = tokens.iter().map(|s| s.to_string()).collect::<HashSet<String>>();
+        let unique_bigrams = bigrams.iter().collect::<HashSet<&String>>();
+        let mut terms = unique_tokens.iter().collect::<Vec<&String>>();
+        terms.extend(unique_bigrams.iter());
+        serde_wasm_bindgen::to_value(&terms).unwrap()
+    }
+
     pub fn add_search_result(&mut self, url: &str, title: &str, extract: &str) {
         self.search_results
             .push(SearchResult::new(url, title, extract));

www/index.js

@@ -11,12 +11,16 @@ let searchButton = document.getElementById("search-button");
 searchButton.addEventListener("click", async (e) => {
   let searchTerm = searchInput.value;
   let ranker = wasm.Ranker.new(searchTerm);
-  let response = await fetch(`https://api.mwmbl.org/api/v1/search/raw?s=${searchTerm}`);
-  let data = await response.json();
-  console.log("Data", data);
-  data.results.forEach((result) => {
-    ranker.add_search_result(result.url, result.title, result.extract);
-  });
+  let terms = ranker.get_query_terms();
+  console.log("Query terms", terms);
+  for (const term of terms) {
+    let response = await fetch(`https://api.mwmbl.org/api/v1/search/raw?s=${term}`);
+    let data = await response.json();
+    console.log("Data", data);
+    for (const result of data.results) {
+      ranker.add_search_result(result.url, result.title, result.extract);
+    }
+  }
   let rankedData = ranker.rank();
   console.log(rankedData);
 
@@ -26,9 +30,10 @@ searchButton.addEventListener("click", async (e) => {
   rankedData.forEach((result) => {
     let div = document.createElement("div");
     div.innerHTML = `
+      <a href="${result.url}">${result.url}</a>
       <h3>${result.title}</h3>
       <p>${result.extract}</p>
-      <a href="${result.url}">result.url</a>
+      <br><br>
     `;
     outputDiv.appendChild(div);
   });