feat: implement multi-source ScanIterator for unified memtable and SS…

engine.rs

@@ -0,0 +1,31 @@
+// engine.rs
+use crate::iterator::ScanIterator;
+
+// ... existing code ...
+
+pub fn scan(&self) -> ScanIterator {
+    ScanIterator::new(&self.memtable, &self.sstables)
+}
+
+pub fn keys(&self) -> Vec<String> {
+    const MAX_SCAN_LIMIT: usize = 1000; // or configurable
+    self.scan()
+        .take(MAX_SCAN_LIMIT)
+        .map(|(key, _)| key)
+        .collect()
+}
+
+pub fn count(&self) -> usize {
+    let memtable_count = self.memtable.len();
+    let sstable_count: usize = self.sstables.iter().map(|s| s.record_count()).sum();
+    memtable_count + sstable_count
+}
+
+// search() also needs updating to use iterator pattern
+pub fn search(&self, query: &str) -> Vec<(String, (Vec<u8>, u128, bool))> {
+    self.scan()
+        .take(1000) // also cap search results
+        .filter(|(_, (_, _, deleted))| !*deleted)
+        .filter(|(_, (value, _, _))| String::from_utf8_lossy(value).contains(query))
+        .collect()
+}

iterator.rs

@@ -0,0 +1,98 @@
+// iterator.rs
+use crate::memtable::MemTable;
+use crate::table::SSTable;
+use std::cmp::Ordering;
+use std::collections::BinaryHeap;
+
+pub struct ScanIterator {
+    heap: BinaryHeap<ScanItem>,
+}
+
+struct ScanItem {
+    key: String,
+    value: (Vec<u8>, u128, bool),
+    source: Source,
+}
+
+enum Source {
+    MemTable,
+    SSTable(usize), // index of SSTable
+}
+
+impl PartialEq for ScanItem {
+    fn eq(&self, other: &Self) -> bool {
+        self.key == other.key
+    }
+}
+
+impl Eq for ScanItem {}
+
+impl PartialOrd for ScanItem {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        // Reverse for min-heap (BinaryHeap is max-heap by default)
+        other.key.partial_cmp(&self.key)
+    }
+}
+
+impl Ord for ScanItem {
+    fn cmp(&self, other: &Self) -> Ordering {
+        other.key.cmp(&self.key)
+    }
+}
+
+impl ScanIterator {
+    pub fn new(memtable: &MemTable, sstables: &[SSTable]) -> Self {
+        let mut heap = BinaryHeap::new();
+
+        // Add all memtable entries
+        for (key, value) in memtable.iter() {
+            heap.push(ScanItem {
+                key: key.clone(),
+                value: value.clone(),
+                source: Source::MemTable,
+            });
+        }
+
+        // Add first entry from each SSTable
+        for (idx, sstable) in sstables.iter().enumerate() {
+            if let Some((key, value)) = sstable.first_key_value() {
+                heap.push(ScanItem {
+                    key: key.clone(),
+                    value: value.clone(),
+                    source: Source::SSTable(idx),
+                });
+            }
+        }
+
+        ScanIterator { heap }
+    }
+}
+
+impl Iterator for ScanIterator {
+    type Item = (String, (Vec<u8>, u128, bool));
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let item = self.heap.pop()?;
+        let key = item.key;
+        let value = item.value;
+
+        // Advance the iterator from which this item came
+        match item.source {
+            Source::MemTable => {
+                // MemTable is already fully in heap, nothing more to add
+            }
+            Source::SSTable(idx) => {
+                // Get next entry from this SSTable
+                if let Some((next_key, next_value)) = SSTable::next_at_index(idx) {
+                    self.heap.push(ScanItem {
+                        key: next_key,
+                        value: next_value,
+                        source: Source::SSTable(idx),
+                    });
+                }
+            }
+        }
+
+        Some((key, value))
+    }
+}

memtable.rs

@@ -0,0 +1,10 @@
+// memtable.rs
+impl MemTable {
+    pub fn iter(&self) -> impl Iterator<Item = (&String, &(Vec<u8>, u128, bool))> {
+        self.map.iter()
+    }
+
+    pub fn len(&self) -> usize {
+        self.map.len()
+    }
+}

src/api/mod.rs

@@ -1,24 +1,49 @@
-use crate::core::engine::{DEFAULT_SCAN_LIMIT, MAX_SCAN_LIMIT};
+pub mod auth;
+pub mod config;
+
+pub use self::config::ServerConfig;
 use crate::LsmEngine;
+use actix_web::{get, web, App, HttpResponse, HttpServer, Responder};
+use serde_json::json;
 
-pub struct ServerConfig;
-impl ServerConfig {
-    pub fn from_file(_path: &str) -> crate::infra::error::Result<Self> {
-        Ok(Self)
-    }
-    pub fn from_env() -> Self {
-        Self
+/// Handler for `GET /keys`.
+/// Returns a JSON object containing an array of all keys (bounded by `MAX_SCAN_LIMIT`).
+#[get("/keys")]
+async fn get_keys(engine: web::Data<LsmEngine>) -> impl Responder {
+    // `LsmEngine::keys` applies the safety bound (MAX_SCAN_LIMIT).
+    match engine.keys() {
+        Ok(keys) => HttpResponse::Ok()
+            .content_type("application/json")
+            .json(json!({ "keys": keys })),
+        Err(e) => {
+            eprintln!("Failed to fetch keys: {:?}", e);
+            HttpResponse::InternalServerError()
+                .content_type("application/json")
+                .json(json!({ "error": "internal server error" }))
+        }
     }
 }
 
-pub async fn start_server(
-    _engine: LsmEngine,
-    _config: ServerConfig,
-) -> crate::infra::error::Result<()> {
-    Ok(())
+/// Register API routes.
+pub fn configure(cfg: &mut web::ServiceConfig) {
+    cfg.service(get_keys);
 }
 
-fn _check_api() {
-    let _ = DEFAULT_SCAN_LIMIT;
-    let _ = MAX_SCAN_LIMIT;
+/// Start the REST API server.
+pub async fn start_server(engine: LsmEngine, config: ServerConfig) -> std::io::Result<()> {
+    let host = config.host.clone();
+    let port = config.port;
+
+    println!("🚀 Starting server at http://{}:{}", host, port);
+
+    let engine_data = web::Data::new(engine);
+
+    HttpServer::new(move || {
+        App::new()
+            .app_data(engine_data.clone())
+            .configure(configure)
+    })
+    .bind((host, port))?
+    .run()
+    .await
 }

src/bin/server.rs

@@ -59,7 +59,9 @@ async fn main() -> std::io::Result<()> {
         .sparse_index_interval(sparse_index_interval)
         .bloom_false_positive_rate(bloom_false_positive_rate)
         .build()
-        .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e.to_string()))?;
+        .map_err(|e: apexstore::LsmError| {
+            io::Error::new(io::ErrorKind::InvalidInput, e.to_string())
+        })?;
 
     // Print LSM configuration
     println!("📋 LSM Engine Configuration:");
@@ -92,5 +94,5 @@ async fn main() -> std::io::Result<()> {
 
     apexstore::api::start_server(engine, server_config)
         .await
-        .map_err(|e| io::Error::other(e.to_string()))
+        .map_err(|e: io::Error| e)
 }

src/bin/tui.rs

@@ -8,7 +8,7 @@
 //!   SEARCH <q> [--prefix] | SCAN <prefix> | ALL | KEYS | COUNT
 //!   STATS [ALL] | BATCH <n> | BATCH SET <file> | DEMO | CLEAR | HELP
 
-use apexstore::{core::engine::LsmStats, infra::config::LsmConfig, LsmEngine};
+use apexstore::{LsmConfig, LsmEngine, LsmError, LsmStats};
 use chrono::Local;
 use crossterm::{
     event::{
@@ -34,6 +34,8 @@ use std::{
 use tui_input::backend::crossterm::EventHandler;
 use tui_input::Input;
 
+type ScanResult = Result<Vec<(Vec<u8>, Vec<u8>)>, LsmError>;
+
 // ─── Palette ──────────────────────────────────────────────────────────────────
 const C_ORANGE: Color = Color::Rgb(255, 110, 30);
 const C_AMBER: Color = Color::Rgb(255, 185, 0);
@@ -170,11 +172,12 @@ impl App {
                 }
                 match self.engine.get(parts[1]) {
                     Ok(Some(v)) => {
+                        let value: Vec<u8> = v;
                         self.log_push(
                             format!(
                                 "\u{2713} '{}' = '{}'",
                                 parts[1],
-                                String::from_utf8_lossy(&v)
+                                String::from_utf8_lossy(&value)
                             ),
                             C_OK,
                         );
@@ -274,46 +277,54 @@ impl App {
             }
 
             // ALL ──────────────────────────────────────────────────────────────
-            "ALL" => match self.engine.scan() {
-                Ok(rows) if rows.is_empty() => self.log_push("\u{26a0}  Database is empty", C_WARN),
-                Ok(rows) => {
-                    self.log_push(format!("\u{2713} {} record(s):", rows.len()), C_OK);
-                    for (k, v) in rows.iter().take(30) {
-                        self.log_push(
-                            format!(
-                                "  {} = {}",
-                                String::from_utf8_lossy(k),
-                                String::from_utf8_lossy(v)
-                            ),
-                            C_TEXT,
-                        );
+            "ALL" => {
+                let res: ScanResult = self.engine.scan();
+                match res {
+                    Ok(rows) if rows.is_empty() => {
+                        self.log_push("\u{26a0}  Database is empty", C_WARN)
                     }
-                    if rows.len() > 30 {
-                        self.log_push(format!("  ... and {} more", rows.len() - 30), C_DIM);
+                    Ok(rows) => {
+                        self.log_push(format!("\u{2713} {} record(s):", rows.len()), C_OK);
+                        for (k, v) in rows.iter().take(30) {
+                            self.log_push(
+                                format!(
+                                    "  {} = {}",
+                                    String::from_utf8_lossy(k),
+                                    String::from_utf8_lossy(v)
+                                ),
+                                C_TEXT,
+                            );
+                        }
+                        if rows.len() > 30 {
+                            self.log_push(format!("  ... and {} more", rows.len() - 30), C_DIM);
+                        }
+                        self.incr_ops();
                     }
-                    self.incr_ops();
+                    Err(e) => self.log_push(format!("\u{274c} {}", e), C_ERR),
                 }
-                Err(e) => self.log_push(format!("\u{274c} {}", e), C_ERR),
-            },
+            }
 
             // KEYS ─────────────────────────────────────────────────────────────
-            "KEYS" => match self.engine.keys() {
-                Ok(keys) if keys.is_empty() => self.log_push("\u{26a0}  No keys found", C_WARN),
-                Ok(keys) => {
-                    self.log_push(format!("\u{2713} {} key(s):", keys.len()), C_OK);
-                    for (i, k) in keys.iter().enumerate().take(30) {
-                        self.log_push(
-                            format!("  {}. {}", i + 1, String::from_utf8_lossy(k)),
-                            C_TEXT,
-                        );
-                    }
-                    if keys.len() > 30 {
-                        self.log_push(format!("  ... and {} more", keys.len() - 30), C_DIM);
+            "KEYS" => {
+                let res: Result<Vec<Vec<u8>>, LsmError> = self.engine.keys();
+                match res {
+                    Ok(keys) if keys.is_empty() => self.log_push("\u{26a0}  No keys found", C_WARN),
+                    Ok(keys) => {
+                        self.log_push(format!("\u{2713} {} key(s):", keys.len()), C_OK);
+                        for (i, k) in keys.iter().enumerate().take(30) {
+                            self.log_push(
+                                format!("  {}. {}", i + 1, String::from_utf8_lossy(k)),
+                                C_TEXT,
+                            );
+                        }
+                        if keys.len() > 30 {
+                            self.log_push(format!("  ... and {} more", keys.len() - 30), C_DIM);
+                        }
+                        self.incr_ops();
                     }
-                    self.incr_ops();
+                    Err(e) => self.log_push(format!("\u{274c} {}", e), C_ERR),
                 }
-                Err(e) => self.log_push(format!("\u{274c} {}", e), C_ERR),
-            },
+            }
 
             // COUNT ────────────────────────────────────────────────────────────
             "COUNT" => match self.engine.count() {
@@ -585,9 +596,9 @@ fn main() -> io::Result<()> {
         .dir_path(PathBuf::from("./.lsm_data"))
         .memtable_max_size(64 * 1024) // 64 KB
         .build()
-        .map_err(|e| io::Error::other(e.to_string()))?;
+        .map_err(|e: LsmError| io::Error::other(e.to_string()))?;
 
-    let engine = LsmEngine::new(config).map_err(|e| io::Error::other(e.to_string()))?;
+    let engine = LsmEngine::new(config).map_err(|e: LsmError| io::Error::other(e.to_string()))?;
 
     let mut terminal = setup()?;
     let mut app = App::new(engine);