diff --git a/.gitignore b/.gitignore index 422a284..142fe63 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ target .lsm_data -/examples/*.json +/local credentials *.sh +.env \ No newline at end of file diff --git a/docs/implementation-plans/global-shared-cache.md b/docs/implementation-plans/global-shared-cache.md new file mode 100644 index 0000000..79033c2 --- /dev/null +++ b/docs/implementation-plans/global-shared-cache.md @@ -0,0 +1,521 @@ +# Plano de Implementação: Global Shared Block Cache + +**Issue:** #35 +**Branch:** `feature/global-shared-cache` +**Prioridade:** HIGH +**Estimativa:** 1-2 dias + +--- + +## 📋 Objetivo + +Implementar um cache de blocos global compartilhado entre todas as instâncias de `SstableReader`, reduzindo o consumo de memória de `O(num_sstables * cache_size)` para `O(cache_size)`. + +## 🎯 Problema Atual + +Cada `SstableReader` possui seu próprio `LruCache>` de tamanho configurado (ex: 64MB). Com múltiplas SSTables abertas: + +``` +100 SSTables × 64MB = 6.4GB de memória +``` + +Isso desperdiça memória e não respeita o limite global de cache configurado. + +## ✅ Solução Proposta + +Criar um cache global único compartilhado via `Arc>` que armazena blocos de todas as SSTables. + +### Arquitetura + +``` +┌─────────────────────────────────────────┐ +│ LsmEngine │ +│ ┌───────────────────────────────────┐ │ +│ │ GlobalBlockCache (Arc) │ │ +│ │ LruCache> │ │ +│ └───────────────────────────────────┘ │ +│ ▲ ▲ ▲ │ +│ │ │ │ │ +│ ┌────────┴────┐ ┌────┴────┐ ┌───┴────┐│ +│ │SSTableReader│ │SSTable │ │SSTable ││ +│ │ (Arc) │ │Reader │ │Reader ││ +│ └─────────────┘ └─────────┘ └────────┘│ +└─────────────────────────────────────────┘ +``` + +--- + +## 📐 Design Detalhado + +### 1. Estrutura `CacheKey` + +**Problema:** Chave atual é apenas `u64` (offset do bloco). Com múltiplos arquivos, colisões são inevitáveis. + +**Solução:** Chave composta identificando arquivo + offset. + +```rust +// src/storage/cache.rs +use std::hash::{Hash, Hasher}; +use std::path::PathBuf; +use std::collections::hash_map::DefaultHasher; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CacheKey { + file_id: u64, // Hash do PathBuf + block_offset: u64, // Offset do bloco no arquivo +} + +impl CacheKey { + pub fn new(path: &PathBuf, offset: u64) -> Self { + let mut hasher = DefaultHasher::new(); + path.hash(&mut hasher); + let file_id = hasher.finish(); + + Self { + file_id, + block_offset: offset, + } + } +} +``` + +**Alternativa considerada:** Usar `PathBuf` diretamente como chave. +- ❌ Overhead de memória (paths podem ser longos) +- ❌ Comparação mais lenta +- ✅ Usar hash é mais eficiente + +### 2. Estrutura `GlobalBlockCache` + +```rust +// src/storage/cache.rs +use lru::LruCache; +use std::num::NonZeroUsize; +use std::sync::{Arc, Mutex}; + +pub struct GlobalBlockCache { + cache: Mutex>>>, +} + +impl GlobalBlockCache { + pub fn new(capacity_mb: usize, block_size: usize) -> Arc { + let capacity_bytes = capacity_mb * 1024 * 1024; + let num_blocks = (capacity_bytes / block_size).max(1); + let capacity = NonZeroUsize::new(num_blocks).unwrap(); + + Arc::new(Self { + cache: Mutex::new(LruCache::new(capacity)), + }) + } + + pub fn get(&self, key: &CacheKey) -> Option>> { + let mut cache = self.cache.lock().unwrap(); + cache.get(key).cloned() + } + + pub fn put(&self, key: CacheKey, value: Vec) { + let mut cache = self.cache.lock().unwrap(); + cache.put(key, Arc::new(value)); + } + + pub fn clear(&self) { + let mut cache = self.cache.lock().unwrap(); + cache.clear(); + } + + // Método para estatísticas (opcional) + pub fn stats(&self) -> CacheStats { + let cache = self.cache.lock().unwrap(); + CacheStats { + len: cache.len(), + cap: cache.cap().get(), + } + } +} + +#[derive(Debug, Clone)] +pub struct CacheStats { + pub len: usize, + pub cap: usize, +} +``` + +**Decisão de Design: `Arc>` em vez de `Vec`** +- ✅ Evita clonagem de dados ao retornar do cache +- ✅ Permite múltiplas referências ao mesmo bloco +- ✅ Cache hit fica O(1) sem cópia + +### 3. Refatoração do `SstableReader` + +**Antes:** +```rust +pub struct SstableReader { + block_cache: LruCache>, // Cache próprio + // ... +} +``` + +**Depois:** +```rust +// src/storage/reader.rs +use crate::storage::cache::{GlobalBlockCache, CacheKey}; + +pub struct SstableReader { + metadata: MetaBlock, + bloom_filter: Bloom<[u8]>, + file: File, + block_cache: Arc, // ✅ Cache compartilhado + path: PathBuf, + config: StorageConfig, +} + +impl SstableReader { + pub fn open( + path: PathBuf, + config: StorageConfig, + block_cache: Arc, // ✅ Injeção de dependência + ) -> Result { + // ... código existente de leitura do arquivo ... + + Ok(Self { + metadata, + bloom_filter, + file, + block_cache, // Usa o cache compartilhado + path, + config, + }) + } + + fn read_block(&mut self, block_meta: &BlockMeta) -> Result> { + let cache_key = CacheKey::new(&self.path, block_meta.offset); + + // Tentar obter do cache + if let Some(cached) = self.block_cache.get(&cache_key) { + return Ok((*cached).clone()); // Arc -> Vec clone + } + + // Cache miss - ler do disco + let block_data = self.read_and_decompress_block(block_meta)?; + + // Armazenar no cache global + self.block_cache.put(cache_key, block_data.clone()); + + Ok(block_data) + } +} +``` + +### 4. Atualização do `LsmEngine` + +```rust +// src/core/engine.rs +use crate::storage::cache::GlobalBlockCache; + +pub struct LsmEngine { + pub(crate) memtable: Mutex, + pub(crate) wal: WriteAheadLog, + pub(crate) sstables: Mutex>, + pub(crate) block_cache: Arc, // ✅ Novo campo + pub(crate) dir_path: PathBuf, + pub(crate) config: LsmConfig, +} + +impl LsmEngine { + pub fn new(config: LsmConfig) -> Result { + std::fs::create_dir_all(&config.core.dir_path)?; + + // ✅ Criar cache global único + let block_cache = GlobalBlockCache::new( + config.storage.block_cache_size_mb, + config.storage.block_size, + ); + + let wal = WriteAheadLog::new(&config.core.dir_path)?; + let wal_records = wal.recover()?; + + let mut sstables = Vec::new(); + for entry in std::fs::read_dir(&config.core.dir_path)? { + let entry = entry?; + let path = entry.path(); + if path.extension().is_some_and(|ext| ext == "sst") { + // ✅ Passar cache para cada reader + match SstableReader::open( + path.clone(), + config.storage.clone(), + Arc::clone(&block_cache), // Compartilhar cache + ) { + Ok(sst) => sstables.push(sst), + Err(e) => warn!("Failed to load SSTable {}: {}", path.display(), e), + } + } + } + + // ... resto do código ... + + Ok(Self { + memtable: Mutex::new(memtable), + wal, + sstables: Mutex::new(sstables), + block_cache, // ✅ Armazenar referência + dir_path: config.core.dir_path.clone(), + config, + }) + } + + fn flush(&self) -> Result<()> { + // ... código de flush ... + + // ✅ Passar cache ao abrir novo SSTable + let reader = SstableReader::open( + sst_path, + self.config.storage.clone(), + Arc::clone(&self.block_cache), + )?; + + // ... + } +} +``` + +--- + +## 🔧 Ordem de Implementação + +### **Fase 1: Estrutura Base** (2-3 horas) + +1. ✅ Criar arquivo `src/storage/cache.rs` +2. ✅ Implementar `CacheKey` com testes unitários +3. ✅ Implementar `GlobalBlockCache` com testes unitários +4. ✅ Adicionar `pub mod cache;` em `src/storage/mod.rs` + +**Testes:** +```rust +#[test] +fn test_cache_key_uniqueness() { + let path1 = PathBuf::from("/data/sst1.sst"); + let path2 = PathBuf::from("/data/sst2.sst"); + + let key1 = CacheKey::new(&path1, 0); + let key2 = CacheKey::new(&path2, 0); + + assert_ne!(key1, key2); // Diferentes arquivos +} + +#[test] +fn test_cache_key_same_file() { + let path = PathBuf::from("/data/sst1.sst"); + + let key1 = CacheKey::new(&path, 0); + let key2 = CacheKey::new(&path, 4096); + + assert_ne!(key1, key2); // Diferentes offsets + assert_eq!(key1.file_id, key2.file_id); // Mesmo arquivo +} + +#[test] +fn test_global_cache_basic() { + let cache = GlobalBlockCache::new(1, 4096); // 1MB, blocos de 4KB + + let key = CacheKey::new(&PathBuf::from("test.sst"), 0); + let data = vec![1, 2, 3, 4]; + + cache.put(key.clone(), data.clone()); + + let retrieved = cache.get(&key).unwrap(); + assert_eq!(*retrieved, data); +} +``` + +### **Fase 2: Refatoração do Reader** (2-3 horas) + +1. ✅ Adicionar campo `Arc` em `SstableReader` +2. ✅ Atualizar assinatura de `SstableReader::open()` +3. ✅ Refatorar `read_block()` para usar `CacheKey` +4. ✅ Remover campo antigo `block_cache: LruCache<...>` +5. ✅ Atualizar método `calculate_cache_capacity()` (não é mais necessário) + +### **Fase 3: Integração na Engine** (1-2 horas) + +1. ✅ Adicionar campo `block_cache` em `LsmEngine` +2. ✅ Criar cache em `LsmEngine::new()` +3. ✅ Passar cache para todos os `SstableReader::open()` +4. ✅ Atualizar método `flush()` para passar cache + +### **Fase 4: Testes de Integração** (2-3 horas) + +```rust +#[test] +fn test_shared_cache_across_sstables() { + let dir = tempdir().unwrap(); + let config = create_test_config(dir.path()); + let cache = GlobalBlockCache::new(1, 4096); + + // Criar múltiplas SSTables + let sst1 = create_test_sstable(dir.path().join("1.sst"), &config, &cache); + let sst2 = create_test_sstable(dir.path().join("2.sst"), &config, &cache); + + // Verificar que ambas usam o mesmo cache + let stats_before = cache.stats(); + + sst1.get("key1").unwrap(); // Popula cache + let stats_after1 = cache.stats(); + assert_eq!(stats_after1.len, stats_before.len + 1); + + sst2.get("key2").unwrap(); // Popula cache + let stats_after2 = cache.stats(); + assert_eq!(stats_after2.len, stats_after1.len + 1); +} + +#[test] +fn test_memory_limit_respected() { + // Criar engine com cache de 1MB + let config = LsmConfig { + storage: StorageConfig { + block_cache_size_mb: 1, + block_size: 4096, + // ... + }, + // ... + }; + + let engine = LsmEngine::new(config).unwrap(); + + // Criar muitas SSTables + for i in 0..100 { + insert_and_flush(&engine, i); + } + + let stats = engine.block_cache.stats(); + let max_blocks = (1 * 1024 * 1024) / 4096; + + // Cache não deve exceder limite + assert!(stats.len <= max_blocks); +} +``` + +### **Fase 5: Benchmarks** (1 hora) + +```rust +// benches/cache_benchmark.rs +use criterion::{black_box, criterion_group, criterion_main, Criterion}; + +fn bench_cache_hit(c: &mut Criterion) { + let cache = GlobalBlockCache::new(64, 4096); + let key = CacheKey::new(&PathBuf::from("test.sst"), 0); + cache.put(key.clone(), vec![0u8; 4096]); + + c.bench_function("cache_hit", |b| { + b.iter(|| { + black_box(cache.get(&key)); + }); + }); +} + +fn bench_cache_miss(c: &mut Criterion) { + let cache = GlobalBlockCache::new(64, 4096); + + c.bench_function("cache_miss", |b| { + b.iter(|| { + let key = CacheKey::new(&PathBuf::from("test.sst"), rand::random()); + black_box(cache.get(&key)); + }); + }); +} +``` + +--- + +## ⚠️ Considerações de Segurança e Performance + +### 1. Contenção de Lock + +**Problema:** `Mutex` pode criar gargalo em workloads com muitos cache hits. + +**Mitigação:** Por enquanto, usar `Mutex` simples. Em otimização futura: +- Considerar `parking_lot::Mutex` (mais rápido) +- Implementar cache sharded (dividir em N sub-caches) + +### 2. Eviction Policy + +**Comportamento:** LRU é justo entre arquivos (não privilegia nenhum arquivo específico). + +**Validação:** Adicionar teste para garantir eviction balanceada. + +### 3. Clonagem de Vec + +**Overhead:** `read_block()` retorna `Vec`, então clonamos o `Arc>`. + +**Alternativa futura:** Retornar `Arc>` diretamente (breaking change na API). + +--- + +## 📊 Métricas de Sucesso + +### Critérios de Aceitação + +- ✅ Cache único compartilhado entre todas as SSTables +- ✅ Uso de memória = `O(cache_size_mb)` independente do número de arquivos +- ✅ Todos os testes unitários e de integração passando +- ✅ Benchmarks mostram overhead < 5% vs cache individual +- ✅ `cargo clippy` sem warnings + +### Métricas de Memória + +**Antes:** +``` +10 SSTables × 64MB = 640MB +100 SSTables × 64MB = 6.4GB +``` + +**Depois:** +``` +10 SSTables → 64MB total +100 SSTables → 64MB total +``` + +**Redução:** 10x para 10 arquivos, 100x para 100 arquivos + +--- + +## 🔄 Compatibilidade + +### Breaking Changes + +✅ **Sim** - A assinatura de `SstableReader::open()` muda: + +```rust +// Antes +SstableReader::open(path, config) + +// Depois +SstableReader::open(path, config, cache) +``` + +### Migração + +Todos os callers de `SstableReader::open()` precisam ser atualizados: +- `LsmEngine::new()` +- `LsmEngine::flush()` +- Testes em `src/storage/reader.rs` + +--- + +## 📚 Referências + +- [LRU Cache in Rust](https://docs.rs/lru/latest/lru/) +- [Arc vs Rc](https://doc.rust-lang.org/std/sync/struct.Arc.html) +- [RocksDB Block Cache](https://github.com/facebook/rocksdb/wiki/Block-Cache) + +--- + +## ✅ Checklist Final + +- [ ] Fase 1: Estrutura base implementada +- [ ] Fase 2: Reader refatorado +- [ ] Fase 3: Engine integrada +- [ ] Fase 4: Testes passando +- [ ] Fase 5: Benchmarks executados +- [ ] Documentação atualizada +- [ ] Code review interno +- [ ] PR criado contra `main` +- [ ] Issue #35 fechada diff --git a/examples/basic.rs b/examples/basic.rs deleted file mode 100644 index 22af5f3..0000000 --- a/examples/basic.rs +++ /dev/null @@ -1,18 +0,0 @@ -use lsm_kv_store::{LsmConfig, LsmEngine}; -use tempfile::tempdir; - -fn main() -> Result<(), Box> { - let dir = tempdir()?; - let cfg = LsmConfig::builder() - .memtable_max_size(4 * 1024) - .dir_path(dir.path().to_path_buf()) - .build()?; - - let db = LsmEngine::new(cfg)?; - db.set("hello".to_string(), b"world".to_vec())?; - - let v = db.get("hello")?; - println!("GET hello = {:?}", v); - - Ok(()) -} diff --git a/examples/demo.rs b/examples/demo.rs deleted file mode 100644 index 11399ae..0000000 --- a/examples/demo.rs +++ /dev/null @@ -1,105 +0,0 @@ -use lsm_kv_store::{LsmConfig, LsmEngine, Result}; -use tempfile::tempdir; - -fn main() -> Result<()> { - let dir = tempdir()?; - let path = dir.path().to_path_buf(); - - // Part 1: Create and populate an LSM-tree database - println!("=== Part 1: Creating LSM-tree database ==="); - let config = LsmConfig::builder() - .dir_path(path.clone()) - .memtable_max_size(1024) - .build()?; - - let db = LsmEngine::new(config)?; - - // Insert some key-value pairs - println!("Inserting keys..."); - db.set("apple".to_string(), b"A red fruit".to_vec())?; - db.set("banana".to_string(), b"A yellow fruit".to_vec())?; - db.set("cherry".to_string(), b"A small red fruit".to_vec())?; - - // Read them back - if let Some(value) = db.get("apple")? { - println!("apple: {}", String::from_utf8_lossy(&value)); - } - - if let Some(value) = db.get("banana")? { - println!("banana: {}", String::from_utf8_lossy(&value)); - } - - // Update a key - println!("\nUpdating 'banana'..."); - db.set("banana".to_string(), b"A VERY yellow fruit".to_vec())?; - - if let Some(value) = db.get("banana")? { - println!("banana (updated): {}", String::from_utf8_lossy(&value)); - } - - // Delete a key - println!("\nDeleting 'cherry'..."); - db.delete("cherry".to_string())?; - - match db.get("cherry")? { - Some(_) => println!("cherry: still exists (unexpected!)"), - None => println!("cherry: deleted"), - } - - // Insert more data to trigger automatic flush - println!("\n=== Part 2: Adding data (automatic flush will occur) ==="); - for i in 0..100 { - let key = format!("key_{:03}", i); - let value = format!("value_{}", i); - db.set(key, value.into_bytes())?; - } - - println!("Data inserted (memtable will flush automatically when full)"); - - // Read some keys - if let Some(value) = db.get("key_042")? { - println!("key_042: {}", String::from_utf8_lossy(&value)); - } - - if let Some(value) = db.get("apple")? { - println!("apple: {}", String::from_utf8_lossy(&value)); - } - - // Part 3: Add more data to create multiple levels - println!("\n=== Part 3: Adding more data ==="); - for i in 100..200 { - let key = format!("key_{:03}", i); - let value = format!("value_{}", i); - db.set(key, value.into_bytes())?; - } - - println!("\nDatabase operations complete."); - println!("Total keys in database: ~200"); - - // Part 4: Reopen the database - println!("\n=== Part 4: Reopening database ==="); - drop(db); - - let config2 = LsmConfig::builder() - .dir_path(path) - .memtable_max_size(1024) - .build()?; - - let db2 = LsmEngine::new(config2)?; - - // Verify data persisted - if let Some(value) = db2.get("apple")? { - println!("apple (after reopen): {}", String::from_utf8_lossy(&value)); - } - - if let Some(value) = db2.get("key_042")? { - println!("key_042 (after reopen): {}", String::from_utf8_lossy(&value)); - } - - if let Some(value) = db2.get("key_150")? { - println!("key_150 (after reopen): {}", String::from_utf8_lossy(&value)); - } - - println!("\n✅ Demo complete!"); - Ok(()) -} diff --git a/examples/generate_high_entropy_data.py b/examples/generate_high_entropy_data.py deleted file mode 100644 index 71a5ac6..0000000 --- a/examples/generate_high_entropy_data.py +++ /dev/null @@ -1,83 +0,0 @@ -import json -import random -import uuid -import hashlib -import string -import os -from datetime import datetime - -def get_random_string(length): - return ''.join(random.choices(string.ascii_letters + string.digits + " .,!?-", k=length)) - -def generate_complex_data(target_mb=10, file_name="stress_test_data.json"): - records = [] - current_size = 0 - target_bytes = target_mb * 1024 * 1024 - - categories = ["identity", "telemetry", "audit_log", "blob_meta", "session_cache"] - - print(f"🚀 Gerando {target_mb}MB de dados com alta entropia...") - - while current_size < target_bytes: - cat = random.choice(categories) - - # --- Cenário 1: Identidade (JSON médio) --- - if cat == "identity": - uid = str(uuid.uuid4()) - key = f"user:profile:{uid[:8]}" - val_obj = { - "uid": uid, - "token": hashlib.sha256(uid.encode()).hexdigest(), - "bio": get_random_string(random.randint(200, 1000)), - "roles": random.sample(["admin", "user", "guest", "manager", "support"], 2), - "active": random.choice([True, False]), - "metadata": {"last_ip": f"{random.randint(1,255)}.{random.randint(1,255)}.0.1"} - } - value = json.dumps(val_obj) - - # --- Cenário 2: Telemetria (Curto e denso) --- - elif cat == "telemetry": - key = f"sensor:th:{random.randint(1000, 9999)}" - value = f"t={random.uniform(18.0, 42.0):.2f};h={random.uniform(30, 90):.2f};st={datetime.now().isoformat()}" - - # --- Cenário 3: Audit Log (Longas strings/Stack traces) --- - elif cat == "audit_log": - key = f"log:{datetime.now().strftime('%Y%m%d')}:{uuid.uuid4().hex[:6]}" - # Simula um erro do sistema com "stack trace" - trace = " | ".join([get_random_string(100) for _ in range(random.randint(10, 50))]) - value = f"ERROR: OutOfMemoryException at {datetime.now()} in Module {get_random_string(10)}. Context: {trace}" - - # --- Cenário 4: Session Cache (Tokens JWT simulados) --- - elif cat == "session_cache": - key = f"sess:{hashlib.md5(str(random.random()).encode()).hexdigest()}" - # Simula um header.payload.signature - header = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" - payload = hashlib.sha512(get_random_string(50).encode()).hexdigest() - sig = hashlib.sha1(get_random_string(20).encode()).hexdigest() - value = f"{header}.{payload}.{sig}" - - # --- Cenário 5: Blob Metadata (Objeto denso) --- - else: - key = f"file:meta:{random.getrandbits(32)}" - value = json.dumps({ - "filename": f"{get_random_string(10)}.pdf", - "checksum": hashlib.md5(get_random_string(100).encode()).hexdigest(), - "tags": [get_random_string(5) for _ in range(10)], - "flags": [random.randint(0, 1000) for _ in range(20)], - "description": get_random_string(random.randint(1000, 5000)) # Valor maior para forçar o flush - }) - - record = {"key": key, "value": value} - records.append(record) - current_size += len(key) + len(value) + 50 # Estimativa de overhead do JSON - - with open(file_name, "w", encoding="utf-8") as f: - json.dump({"records": records}, f, ensure_ascii=False) - - final_size = os.path.getsize(file_name) / (1024 * 1024) - print(f"✅ Arquivo '{file_name}' criado.") - print(f"📊 Total de registros: {len(records)}") - print(f"📦 Tamanho final: {final_size:.2f} MB") - -if __name__ == "__main__": - generate_complex_data(10) # 10MB para garantir o estouro da memtable de 4MB \ No newline at end of file diff --git a/requests/api.rest b/requests/api.rest deleted file mode 100644 index 97ce414..0000000 --- a/requests/api.rest +++ /dev/null @@ -1,457 +0,0 @@ -### ============================================================================ -### LSM-Tree Key-Value Store - REST API -### Extensão: REST Client (VS Code) -### ============================================================================ - -### Variáveis globais -#@baseUrl = http://localhost:8080 -@baseUrl = https://steadfast-connection-dev.up.railway.app -@contentType = application/json - -### ============================================================================ -### 1. HEALTHCHECK & STATUS -### ============================================================================ - -### Health Check -# @name healthCheck -GET {{baseUrl}}/health - -### - -### Estatísticas do Engine -# @name stats -GET {{baseUrl}}/stats/all - -### ============================================================================ -### 2. OPERAÇÕES BÁSICAS (CRUD Único) -### ============================================================================ - -### Inserir chave simples -# @name setKey1 -POST {{baseUrl}}/keys -Content-Type: {{contentType}} - -{ - "key": "feature:user:texuguito", - "value": "{\"name\":\"Elio Neto\",\"email\":\"netoo.elio@hotmail.com\"}" -} - -### - -### Inserir produto -# @name setProduct -POST {{baseUrl}}/keys -Content-Type: {{contentType}} - -{ - "key": "product:laptop-001", - "value": "Dell Inspiron 15, 16GB RAM, 512GB SSD" -} - -### - -### Buscar chave específica -# @name getKey -GET {{baseUrl}}/keys/feature:user:texuguito - -### - -### Buscar produto -# @name getProduct -GET {{baseUrl}}/keys/product:laptop-001 - -### - -### Atualizar chave (sobrescrever) -# @name updateKey -POST {{baseUrl}}/keys -Content-Type: {{contentType}} - -{ - "key": "user:alice", - "value": "Alice Silva Santos (atualizado)" -} - -### - -### Deletar chave -# @name deleteKey -DELETE {{baseUrl}}/keys/user:alice - -### - -### Verificar chave deletada -# @name verifyDeleted -GET {{baseUrl}}/keys/user:alice1 - -### ============================================================================ -### 3. OPERAÇÕES BATCH (Múltiplos registros) -### ============================================================================ - -### Inserir múltiplos usuários (BATCH INSERT) -# @name batchInsertUsers -POST {{baseUrl}}/keys/batch -Content-Type: {{contentType}} - -{ - "records": [ - {"key": "user:1", "value": "Alice Silva"}, - {"key": "user:2", "value": "Bob Santos"}, - {"key": "user:3", "value": "Charlie Costa"}, - {"key": "user:4", "value": "Diana Oliveira"}, - {"key": "user:5", "value": "Eduardo Lima"} - ] -} - -### - -### Inserir múltiplos produtos (BATCH INSERT) -# @name batchInsertProducts -POST {{baseUrl}}/keys/batch -Content-Type: {{contentType}} - -{ - "records": [ - {"key": "product:100", "value": "Notebook Dell"}, - {"key": "product:101", "value": "Mouse Logitech"}, - {"key": "product:102", "value": "Teclado Mecânico"}, - {"key": "product:103", "value": "Monitor LG 27\""}, - {"key": "product:104", "value": "Webcam HD"} - ] -} - -### - -### Inserir pedidos (BATCH INSERT) -# @name batchInsertOrders -POST {{baseUrl}}/keys/batch -Content-Type: {{contentType}} - -{ - "records": [ - {"key": "order:2024-001", "value": "Alice - Notebook - R$ 3500"}, - {"key": "order:2024-002", "value": "Bob - Mouse - R$ 150"}, - {"key": "order:2024-003", "value": "Charlie - Teclado - R$ 450"}, - {"key": "order:2025-001", "value": "Diana - Monitor - R$ 1200"}, - {"key": "order:2025-002", "value": "Eduardo - Webcam - R$ 300"} - ] -} - -### - -### Deletar múltiplas chaves (BATCH DELETE) -# @name batchDelete -DELETE {{baseUrl}}/keys/batch -Content-Type: {{contentType}} - -{ - "keys": ["user:3", "user:5", "product:104"] -} - -### - -### Deletar todos os pedidos de 2024 (preparar batch) -# @name batchDeleteOrders2024 -DELETE {{baseUrl}}/keys/batch -Content-Type: {{contentType}} - -{ - "keys": ["order:2024-001", "order:2024-002", "order:2024-003"] -} - -### ============================================================================ -### 4. BUSCA & LISTAGEM -### ============================================================================ - -### Listar todas as chaves -# @name listAllKeys -GET {{baseUrl}}/keys - -### - -### Scan completo (chaves + valores) -# @name scanAll -GET {{baseUrl}}/scan - -### - -### Buscar por substring "user" (qualquer parte da chave) -# @name searchUser -GET {{baseUrl}}/keys/search?q=user - -### - -### Buscar por substring "product" -# @name searchProduct -GET {{baseUrl}}/keys/search?q=product - -### - -### Buscar por prefixo "user:" (início da chave) -# @name searchPrefixUser -GET {{baseUrl}}/keys/search?q=user:&prefix=true - -### - -### Buscar por prefixo "product:" -# @name searchPrefixProduct -GET {{baseUrl}}/keys/search?q=product:&prefix=true - -### - -### Buscar por prefixo "order:2024" -# @name searchPrefix2024Orders -GET {{baseUrl}}/keys/search?q=order:2024&prefix=true - -### - -### Buscar por prefixo "order:2025" -# @name searchPrefix2025Orders -GET {{baseUrl}}/keys/search?q=order:2025&prefix=true - -### ============================================================================ -### 5. TESTES DE CARGA & PERFORMANCE -### ============================================================================ - -### Inserir 100 registros em batch (teste de carga) -# @name loadTest100 -POST {{baseUrl}}/keys/batch -Content-Type: {{contentType}} - -{ - "records": [ - {"key": "test:001", "value": "Test value 001"}, - {"key": "test:002", "value": "Test value 002"}, - {"key": "test:003", "value": "Test value 003"}, - {"key": "test:004", "value": "Test value 004"}, - {"key": "test:005", "value": "Test value 005"}, - {"key": "test:006", "value": "Test value 006"}, - {"key": "test:007", "value": "Test value 007"}, - {"key": "test:008", "value": "Test value 008"}, - {"key": "test:009", "value": "Test value 009"}, - {"key": "test:010", "value": "Test value 010"} - ] -} - -### - -### Buscar todos os testes -# @name searchAllTests -GET {{baseUrl}}/keys/search?q=sensor:&prefix=true - -### - -### Limpar registros de teste -# @name cleanupTests -DELETE {{baseUrl}}/keys/batch -Content-Type: {{contentType}} - -{ - "keys": [ - "test:001", "test:002", "test:003", "test:004", "test:005", - "test:006", "test:007", "test:008", "test:009", "test:010" - ] -} - -### ============================================================================ -### 6. CENÁRIOS PRÁTICOS -### ============================================================================ - -### Cenário 1: E-commerce - Cadastro de produtos -# @name ecommerce_products -POST {{baseUrl}}/keys/batch -Content-Type: {{contentType}} - -{ - "records": [ - {"key": "catalog:electronics:laptop-dell-i5", "value": "{\"name\":\"Dell Inspiron i5\",\"price\":3200,\"stock\":15}"}, - {"key": "catalog:electronics:laptop-hp-i7", "value": "{\"name\":\"HP Pavilion i7\",\"price\":4500,\"stock\":8}"}, - {"key": "catalog:electronics:mouse-logitech", "value": "{\"name\":\"Logitech MX Master\",\"price\":350,\"stock\":50}"}, - {"key": "catalog:books:clean-code", "value": "{\"name\":\"Clean Code\",\"author\":\"Robert Martin\",\"price\":85}"}, - {"key": "catalog:books:design-patterns", "value": "{\"name\":\"Design Patterns\",\"author\":\"Gang of Four\",\"price\":120}"} - ] -} - -### - -### Buscar todos os eletrônicos -# @name search_electronics -GET {{baseUrl}}/keys/search?q=catalog:electronics:&prefix=true - -### - -### Buscar todos os livros -# @name search_books -GET {{baseUrl}}/keys/search?q=catalog:books:&prefix=true - -### - -### Cenário 2: Sistema de cache - Sessões de usuário -# @name cache_sessions -POST {{baseUrl}}/keys/batch -Content-Type: {{contentType}} - -{ - "records": [ - {"key": "session:abc123", "value": "{\"user_id\":1,\"email\":\"alice@example.com\",\"expires\":1706100000}"}, - {"key": "session:def456", "value": "{\"user_id\":2,\"email\":\"bob@example.com\",\"expires\":1706110000}"}, - {"key": "session:ghi789", "value": "{\"user_id\":3,\"email\":\"charlie@example.com\",\"expires\":1706120000}"} - ] -} - -### - -### Buscar todas as sessões ativas -# @name search_sessions -GET {{baseUrl}}/keys/search?q=session:&prefix=true - -### - -### Expirar sessões (deletar) -# @name expire_sessions -DELETE {{baseUrl}}/keys/batch -Content-Type: {{contentType}} - -{ - "keys": ["session:abc123", "session:def456"] -} - -### - -### Verificar sessões restantes -# @name check_remaining_sessions -GET {{baseUrl}}/keys/search?q=session:&prefix=true - -### ============================================================================ -### 7. CLEANUP GERAL -### ============================================================================ - -### Ver estatísticas antes da limpeza -# @name stats_before_cleanup -GET {{baseUrl}}/stats - -### - -### Listar todas as chaves para review -# @name list_all_before_cleanup -GET {{baseUrl}}/keys - -### - -### ATENÇÃO: Deletar todos os registros de teste -### (Ajuste as chaves conforme necessário) -# @name cleanup_all -DELETE {{baseUrl}}/keys/batch -Content-Type: {{contentType}} - -{ - "keys": [ - "user:1", "user:2", "user:4", - "product:100", "product:101", "product:102", "product:103", - "order:2025-001", "order:2025-002", - "catalog:electronics:laptop-dell-i5", - "catalog:electronics:laptop-hp-i7", - "catalog:electronics:mouse-logitech", - "catalog:books:clean-code", - "catalog:books:design-patterns", - "session:ghi789" - ] -} - -### - -### Ver estatísticas após limpeza -# @name stats_after_cleanup -GET {{baseUrl}}/stats - -### - -### Verificar banco vazio -# @name verify_empty -GET {{baseUrl}}/scan - - -### ============================================================================ -### 6. CENÁRIOS CARGA & PERFORMANCE -### ============================================================================ - -# @name Carga -POST {{baseUrl}}/keys/batch -Content-Type: {{contentType}} - -{ - "records": [ - // --- USUÁRIOS (70 registros) --- - {"key": "user:6", "value": "Fabio Vieira"}, {"key": "user:7", "value": "Gabriel Souza"}, {"key": "user:8", "value": "Helena Matos"}, {"key": "user:9", "value": "Isabela Rocha"}, {"key": "user:10", "value": "João Pereira"}, - {"key": "user:11", "value": "Karina Mendes"}, {"key": "user:12", "value": "Lucas Fernandes"}, {"key": "user:13", "value": "Marina Silva"}, {"key": "user:14", "value": "Natan Costa"}, {"key": "user:15", "value": "Olívia Gomes"}, - {"key": "user:16", "value": "Paulo Ricardo"}, {"key": "user:17", "value": "Quênia Lopes"}, {"key": "user:18", "value": "Rafael Alves"}, {"key": "user:19", "value": "Sara Monte"}, {"key": "user:20", "value": "Tiago Luz"}, - {"key": "user:21", "value": "Ursula Bezerra"}, {"key": "user:22", "value": "Vitor Hugo"}, {"key": "user:23", "value": "Wagner Moura"}, {"key": "user:24", "value": "Xavier Neto"}, {"key": "user:25", "value": "Yara Flor"}, - {"key": "user:26", "value": "Zeca Pagode"}, {"key": "user:27", "value": "Ana Clara"}, {"key": "user:28", "value": "Bruno Henrique"}, {"key": "user:29", "value": "Caio Castro"}, {"key": "user:30", "value": "Daniela Mercury"}, - {"key": "user:31", "value": "Elaine Santos"}, {"key": "user:32", "value": "Fernando Pessoa"}, {"key": "user:33", "value": "Gisele Bündchen"}, {"key": "user:34", "value": "Heitor Villa"}, {"key": "user:35", "value": "Igor Kanário"}, - {"key": "user:36", "value": "Júlia Roberts"}, {"key": "user:37", "value": "Kléber Glad"}, {"key": "user:38", "value": "Leandro Karnal"}, {"key": "user:39", "value": "Marta Vieira"}, {"key": "user:40", "value": "Nelson Motta"}, - {"key": "user:41", "value": "Otávio Mesquita"}, {"key": "user:42", "value": "Priscila Fantin"}, {"key": "user:43", "value": "Queiroz Galvão"}, {"key": "user:44", "value": "Renato Aragão"}, {"key": "user:45", "value": "Sônia Abrão"}, - {"key": "user:46", "value": "Tadeu Schmidt"}, {"key": "user:47", "value": "Ulysses Guimarães"}, {"key": "user:48", "value": "Valéria Almeida"}, {"key": "user:49", "value": "Wilson Simonal"}, {"key": "user:50", "value": "Xuxa Meneghel"}, - {"key": "user:51", "value": "Yuri Gagarin"}, {"key": "user:52", "value": "Zico Galo"}, {"key": "user:53", "value": "Amanda Nunes"}, {"key": "user:54", "value": "Beto Carrero"}, {"key": "user:55", "value": "Ciro Gomes"}, - {"key": "user:56", "value": "Dadá Maravilha"}, {"key": "user:57", "value": "Eder Jofre"}, {"key": "user:58", "value": "Fafá de Belém"}, {"key": "user:59", "value": "Gal Costa"}, {"key": "user:60", "value": "Huguinho Silva"}, - {"key": "user:61", "value": "Ivan Lins"}, {"key": "user:62", "value": "Jorge Ben"}, {"key": "user:63", "value": "Kátia Cega"}, {"key": "user:64", "value": "Lulu Santos"}, {"key": "user:65", "value": "Milton Nascimento"}, - {"key": "user:66", "value": "Ney Matogrosso"}, {"key": "user:67", "value": "Oscar Niemeyer"}, {"key": "user:68", "value": "Pixinguinha Silva"}, {"key": "user:69", "value": "Quico Bola"}, {"key": "user:70", "value": "Raul Seixas"}, - {"key": "user:71", "value": "Sandra de Sá"}, {"key": "user:72", "value": "Tiririca Oliveira"}, {"key": "user:73", "value": "Ubirajara Indio"}, {"key": "user:74", "value": "Vampeta Veloso"}, {"key": "user:75", "value": "Zumbi Palmares"}, - - // --- PRODUTOS (65 registros) --- - {"key": "product:105", "value": "Smartphone Samsung S23"}, {"key": "product:106", "value": "iPhone 15 Pro"}, {"key": "product:107", "value": "Carregador Portátil"}, {"key": "product:108", "value": "Cabo USB-C 2m"}, {"key": "product:109", "value": "Fone Bluetooth JBL"}, - {"key": "product:110", "value": "Mouse Pad Gamer"}, {"key": "product:111", "value": "Suporte Articulado"}, {"key": "product:112", "value": "Cadeira Ergonômica"}, {"key": "product:113", "value": "Mesa de Escritório"}, {"key": "product:114", "value": "Luminária LED"}, - {"key": "product:115", "value": "HD Externo 1TB"}, {"key": "product:116", "value": "SSD 500GB NVMe"}, {"key": "product:117", "value": "Memória RAM 16GB"}, {"key": "product:118", "value": "Placa de Vídeo RTX"}, {"key": "product:119", "value": "Processador Intel i7"}, - {"key": "product:120", "value": "Cooler Master"}, {"key": "product:121", "value": "Gabinete ATX"}, {"key": "product:122", "value": "Fonte 600W 80 Plus"}, {"key": "product:123", "value": "Roteador Wi-Fi 6"}, {"key": "product:124", "value": "Repetidor de Sinal"}, - {"key": "product:125", "value": "Kindle Paperwhite"}, {"key": "product:126", "value": "Tablet iPad Air"}, {"key": "product:127", "value": "Smartwatch Series 9"}, {"key": "product:128", "value": "Caixa de Som Echo Dot"}, {"key": "product:129", "value": "Fire TV Stick 4K"}, - {"key": "product:130", "value": "Microfone Condensador"}, {"key": "product:131", "value": "Interface de Áudio"}, {"key": "product:132", "value": "Controladora DJ"}, {"key": "product:133", "value": "Pedal de Guitarra"}, {"key": "product:134", "value": "Violão Elétrico"}, - {"key": "product:135", "value": "Bateria Eletrônica"}, {"key": "product:136", "value": "Câmera Canon EOS"}, {"key": "product:137", "value": "Lente 50mm f1.8"}, {"key": "product:138", "value": "Tripé Profissional"}, {"key": "product:139", "value": "Cartão SD 128GB"}, - {"key": "product:140", "value": "Drone DJI Mini"}, {"key": "product:141", "value": "Impressora Laser"}, {"key": "product:142", "value": "Scanner de Mesa"}, {"key": "product:143", "value": "Projetor 4K"}, {"key": "product:144", "value": "Tela de Projeção"}, - {"key": "product:145", "value": "Nobreak 1200VA"}, {"key": "product:146", "value": "Filtro de Linha"}, {"key": "product:147", "value": "Hub USB 3.0"}, {"key": "product:148", "value": "Adaptador HDMI"}, {"key": "product:149", "value": "Teclado Numérico"}, - {"key": "product:150", "value": "Mochila para Laptop"}, {"key": "product:151", "value": "Pasta Executiva"}, {"key": "product:152", "value": "Caneta Digitalizadora"}, {"key": "product:153", "value": "Mesa Digitalizadora"}, {"key": "product:154", "value": "Calculadora HP 12C"}, - {"key": "product:155", "value": "Fragmentadora de Papel"}, {"key": "product:156", "value": "Gaveteiro de Aço"}, {"key": "product:157", "value": "Quadro Branco"}, {"key": "product:158", "value": "Organizador de Cabos"}, {"key": "product:159", "value": "Pilha Recarregável"}, - {"key": "product:160", "value": "Multímetro Digital"}, {"key": "product:161", "value": "Ferro de Solda"}, {"key": "product:162", "value": "Parafusadeira Bosch"}, {"key": "product:163", "value": "Jogo de Chaves"}, {"key": "product:164", "value": "Maleta de Ferramentas"}, - {"key": "product:165", "value": "Ar Condicionado 12kBTU"}, {"key": "product:166", "value": "Ventilador de Torre"}, {"key": "product:167", "value": "Umidificador de Ar"}, {"key": "product:168", "value": "Purificador de Água"}, {"key": "product:169", "value": "Cafeteira Nespresso"}, - - // --- PEDIDOS (65 registros) --- - {"key": "order:2025-003", "value": "Fabio - Smartphone - R$ 4200"}, {"key": "order:2025-004", "value": "Gabriel - iPhone - R$ 7500"}, {"key": "order:2025-005", "value": "Helena - Carregador - R$ 120"}, {"key": "order:2025-006", "value": "Isabela - Cabo USB - R$ 50"}, {"key": "order:2025-007", "value": "João - Fone JBL - R$ 350"}, - {"key": "order:2025-008", "value": "Karina - Mouse Pad - R$ 80"}, {"key": "order:2025-009", "value": "Lucas - Suporte - R$ 150"}, {"key": "order:2025-010", "value": "Marina - Cadeira - R$ 1200"}, {"key": "order:2025-011", "value": "Natan - Mesa - R$ 850"}, {"key": "order:2025-012", "value": "Olívia - Luminária - R$ 90"}, - {"key": "order:2025-013", "value": "Paulo - HD Externo - R$ 400"}, {"key": "order:2025-014", "value": "Quênia - SSD - R$ 300"}, {"key": "order:2025-015", "value": "Rafael - Memória RAM - R$ 450"}, {"key": "order:2025-016", "value": "Sara - Placa Vídeo - R$ 2800"}, {"key": "order:2025-017", "value": "Tiago - Processador - R$ 1500"}, - {"key": "order:2025-018", "value": "Ursula - Cooler - R$ 200"}, {"key": "order:2025-019", "value": "Vitor - Gabinete - R$ 350"}, {"key": "order:2025-020", "value": "Wagner - Fonte - R$ 450"}, {"key": "order:2025-021", "value": "Xavier - Roteador - R$ 600"}, {"key": "order:2025-022", "value": "Yara - Repetidor - R$ 150"}, - {"key": "order:2025-023", "value": "Zeca - Kindle - R$ 500"}, {"key": "order:2025-024", "value": "Ana - Tablet - R$ 3200"}, {"key": "order:2025-025", "value": "Bruno - Smartwatch - R$ 2100"}, {"key": "order:2025-026", "value": "Caio - Echo Dot - R$ 350"}, {"key": "order:2025-027", "value": "Daniela - Fire TV - R$ 280"}, - {"key": "order:2025-028", "value": "Elaine - Microfone - R$ 900"}, {"key": "order:2025-029", "value": "Fernando - Interface - R$ 1200"}, {"key": "order:2025-030", "value": "Gisele - Controladora - R$ 2500"}, {"key": "order:2025-031", "value": "Heitor - Pedal - R$ 600"}, {"key": "order:2025-032", "value": "Igor - Violão - R$ 1800"}, - {"key": "order:2025-033", "value": "Júlia - Bateria - R$ 4500"}, {"key": "order:2025-034", "value": "Kléber - Câmera - R$ 6000"}, {"key": "order:2025-035", "value": "Leandro - Lente - R$ 1100"}, {"key": "order:2025-036", "value": "Marta - Tripé - R$ 400"}, {"key": "order:2025-037", "value": "Nelson - Cartão SD - R$ 180"}, - {"key": "order:2025-038", "value": "Otávio - Drone - R$ 3800"}, {"key": "order:2025-039", "value": "Priscila - Impressora - R$ 1200"}, {"key": "order:2025-040", "value": "Queiroz - Scanner - R$ 700"}, {"key": "order:2025-041", "value": "Renato - Projetor - R$ 2500"}, {"key": "order:2025-042", "value": "Sônia - Tela - R$ 600"}, - {"key": "order:2025-043", "value": "Tadeu - Nobreak - R$ 950"}, {"key": "order:2025-044", "value": "Ulysses - Filtro - R$ 70"}, {"key": "order:2025-045", "value": "Valéria - Hub USB - R$ 120"}, {"key": "order:2025-046", "value": "Wilson - Adaptador - R$ 50"}, {"key": "order:2025-047", "value": "Xuxa - Teclado Num - R$ 90"}, - {"key": "order:2025-048", "value": "Yuri - Mochila - R$ 250"}, {"key": "order:2025-049", "value": "Zico - Pasta - R$ 180"}, {"key": "order:2025-050", "value": "Amanda - Caneta Dig - R$ 400"}, {"key": "order:2025-051", "value": "Beto - Mesa Dig - R$ 1500"}, {"key": "order:2025-052", "value": "Ciro - Calculadora - R$ 600"}, - {"key": "order:2025-053", "value": "Dadá - Fragmentadora - R$ 450"}, {"key": "order:2025-054", "value": "Eder - Gaveteiro - R$ 300"}, {"key": "order:2025-055", "value": "Fafá - Quadro - R$ 150"}, {"key": "order:2025-056", "value": "Gal - Organizador - R$ 40"}, {"key": "order:2025-057", "value": "Huguinho - Pilha - R$ 80"}, - {"key": "order:2025-058", "value": "Ivan - Multímetro - R$ 120"}, {"key": "order:2025-059", "value": "Jorge - Ferro Solda - R$ 60"}, {"key": "order:2025-060", "value": "Kátia - Parafusadeira - R$ 350"}, {"key": "order:2025-061", "value": "Lulu - Jogo Chaves - R$ 200"}, {"key": "order:2025-062", "value": "Milton - Maleta - R$ 400"}, - {"key": "order:2025-063", "value": "Ney - Ar Cond - R$ 2200"}, {"key": "order:2025-064", "value": "Oscar - Ventilador - R$ 250"}, {"key": "order:2025-065", "value": "Pixinguinha - Umidificador - R$ 180"}, {"key": "order:2025-066", "value": "Quico - Purificador - R$ 600"}, {"key": "order:2025-067", "value": "Raul - Cafeteira - R$ 450"} - - ] -} - -### -# @name cargo_test_stats -GET {{baseUrl}}/stats - -### -# @name cargo_test_stats -GET {{baseUrl}}/stats/all -### - -### Verificar banco vazio -# @name cargo_test_scan -GET {{baseUrl}}/scan - -### demora cerca de 172500 ms. Dependendo da máquina, pode variar. Adiciona 6600 registros ao banco. -# -POST {{baseUrl}}/keys/batch -Content-Type: application/json - -< ../examples/stress_test_data.json -### -# -POST {{baseUrl}}/keys/batch -Content-Type: application/json - -< ../examples/carga.json \ No newline at end of file diff --git a/requests/features.rest b/requests/features.rest deleted file mode 100644 index dafb757..0000000 --- a/requests/features.rest +++ /dev/null @@ -1,27 +0,0 @@ -### Variáveis globais -#@baseUrl = http://localhost:8080 -@baseUrl = https://steadfast-connection-dev.up.railway.app -@contentType = application/json - -### ============================================================================ -### Listar todas as feature flags -# @name listFeatures -GET {{baseUrl}}/features -### ============================================================================ -### Obter status de uma feature flag específica -# @name getFeature -GET {{baseUrl}}/features/feature:admin:frontend -### ============================================================================ -### criar ou editar uma features flag -# @name createOrUpdateFeature -POST {{baseUrl}}/features/admin:frontend:menu:keys -Content-Type: {{contentType}} - -{ - "enabled": true, - "description": "Habilita o menu de scan no frontend" -} -### ============================================================================ -### Buscar chave específica -# @name getKey -GET {{baseUrl}}/keys/user:alice \ No newline at end of file diff --git a/src/core/engine.rs b/src/core/engine.rs index cf96581..f390280 100644 --- a/src/core/engine.rs +++ b/src/core/engine.rs @@ -3,12 +3,13 @@ use crate::core::memtable::MemTable; use crate::infra::config::LsmConfig; use crate::infra::error::{LsmError, Result}; use crate::storage::builder::SstableBuilder; +use crate::storage::cache::GlobalBlockCache; use crate::storage::reader::SstableReader; use crate::storage::wal::WriteAheadLog; use std::collections::HashMap; use std::path::PathBuf; -use std::sync::{Mutex, MutexGuard}; +use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{SystemTime, UNIX_EPOCH}; use serde::Serialize; @@ -30,6 +31,7 @@ pub struct LsmEngine { pub(crate) memtable: Mutex, pub(crate) wal: WriteAheadLog, pub(crate) sstables: Mutex>, + pub(crate) block_cache: Arc, pub(crate) dir_path: PathBuf, pub(crate) config: LsmConfig, } @@ -38,6 +40,12 @@ impl LsmEngine { pub fn new(config: LsmConfig) -> Result { std::fs::create_dir_all(&config.core.dir_path)?; + // Create global shared block cache + let block_cache = GlobalBlockCache::new( + config.storage.block_cache_size_mb, + config.storage.block_size, + ); + let wal = WriteAheadLog::new(&config.core.dir_path)?; let wal_records = wal.recover()?; @@ -46,7 +54,11 @@ impl LsmEngine { let entry = entry?; let path = entry.path(); if path.extension().is_some_and(|ext| ext == "sst") { - match SstableReader::open(path.clone(), config.storage.clone()) { + match SstableReader::open( + path.clone(), + config.storage.clone(), + Arc::clone(&block_cache), + ) { Ok(sst) => sstables.push(sst), Err(e) => warn!("Failed to load SSTable {}: {}", path.display(), e), } @@ -62,15 +74,17 @@ impl LsmEngine { } info!( - "LSM Engine initialized: {} sstables, memtable={} records", + "LSM Engine initialized: {} sstables, memtable={} records, cache={}MB", sstables.len(), - memtable.data.len() + memtable.data.len(), + config.storage.block_cache_size_mb ); Ok(Self { memtable: Mutex::new(memtable), wal, sstables: Mutex::new(sstables), + block_cache, dir_path: config.core.dir_path.clone(), config, }) @@ -200,8 +214,12 @@ impl LsmEngine { } let sst_path = builder.finish()?; - // Open the new SSTable as Reader (V2) - let reader = SstableReader::open(sst_path, self.config.storage.clone())?; + // Open the new SSTable as Reader (V2) with shared cache + let reader = SstableReader::open( + sst_path, + self.config.storage.clone(), + Arc::clone(&self.block_cache), + )?; let mut sstables = self.sstables_lock()?; sstables.insert(0, reader); @@ -281,11 +299,15 @@ impl LsmEngine { Err(e) => return format!("LSM Stats error: {e}"), }; + let cache_stats = self.block_cache.stats(); + format!( - "LSM Stats:\n MemTable: {} records, ~{} KB\n SSTables: {} files", + "LSM Stats:\n MemTable: {} records, ~{} KB\n SSTables: {} files\n Cache: {}/{} blocks", memtable.data.len(), memtable.size_bytes / 1024, - sstables.len() + sstables.len(), + cache_stats.len, + cache_stats.cap ) } diff --git a/src/storage/cache.rs b/src/storage/cache.rs new file mode 100644 index 0000000..e82788c --- /dev/null +++ b/src/storage/cache.rs @@ -0,0 +1,277 @@ +use lru::LruCache; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::num::NonZeroUsize; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +/// Cache key that uniquely identifies a block across multiple SSTable files. +/// Combines file identity (hash of path) with block offset. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CacheKey { + file_id: u64, // Hash of the file path + block_offset: u64, // Block offset within the file +} + +impl CacheKey { + /// Creates a new cache key from a file path and block offset. + /// + /// # Arguments + /// * `path` - Path to the SSTable file + /// * `offset` - Byte offset of the block within the file + pub fn new(path: &PathBuf, offset: u64) -> Self { + let mut hasher = DefaultHasher::new(); + path.hash(&mut hasher); + let file_id = hasher.finish(); + + Self { + file_id, + block_offset: offset, + } + } +} + +/// Global shared block cache that is shared across all SSTable readers. +/// Uses LRU eviction policy to manage memory usage. +#[derive(Debug)] +pub struct GlobalBlockCache { + cache: Mutex>>>, +} + +impl GlobalBlockCache { + /// Creates a new global block cache. + /// + /// # Arguments + /// * `capacity_mb` - Maximum cache size in megabytes + /// * `block_size` - Size of each block in bytes + /// + /// # Returns + /// Arc-wrapped cache instance for shared ownership + pub fn new(capacity_mb: usize, block_size: usize) -> Arc { + let capacity_bytes = capacity_mb * 1024 * 1024; + let num_blocks = (capacity_bytes / block_size).max(1); + let capacity = NonZeroUsize::new(num_blocks).unwrap(); + + Arc::new(Self { + cache: Mutex::new(LruCache::new(capacity)), + }) + } + + /// Retrieves a block from the cache. + /// + /// # Arguments + /// * `key` - Cache key identifying the block + /// + /// # Returns + /// Some(Arc>) if found, None if cache miss + pub fn get(&self, key: &CacheKey) -> Option>> { + let mut cache = self.cache.lock().unwrap(); + cache.get(key).cloned() + } + + /// Inserts a block into the cache. + /// + /// # Arguments + /// * `key` - Cache key identifying the block + /// * `value` - Block data to cache + pub fn put(&self, key: CacheKey, value: Vec) { + let mut cache = self.cache.lock().unwrap(); + cache.put(key, Arc::new(value)); + } + + /// Clears all entries from the cache. + pub fn clear(&self) { + let mut cache = self.cache.lock().unwrap(); + cache.clear(); + } + + /// Returns cache statistics. + pub fn stats(&self) -> CacheStats { + let cache = self.cache.lock().unwrap(); + CacheStats { + len: cache.len(), + cap: cache.cap().get(), + } + } +} + +/// Statistics about cache usage. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CacheStats { + /// Number of entries currently in cache + pub len: usize, + /// Maximum capacity of the cache + pub cap: usize, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_cache_key_uniqueness_different_files() { + let path1 = PathBuf::from("/data/sst1.sst"); + let path2 = PathBuf::from("/data/sst2.sst"); + + let key1 = CacheKey::new(&path1, 0); + let key2 = CacheKey::new(&path2, 0); + + // Different files should produce different keys + assert_ne!(key1, key2); + assert_ne!(key1.file_id, key2.file_id); + } + + #[test] + fn test_cache_key_same_file_different_offsets() { + let path = PathBuf::from("/data/sst1.sst"); + + let key1 = CacheKey::new(&path, 0); + let key2 = CacheKey::new(&path, 4096); + + // Different offsets should produce different keys + assert_ne!(key1, key2); + // But same file_id + assert_eq!(key1.file_id, key2.file_id); + } + + #[test] + fn test_cache_key_deterministic() { + let path = PathBuf::from("/data/test.sst"); + + let key1 = CacheKey::new(&path, 1024); + let key2 = CacheKey::new(&path, 1024); + + // Same path and offset should produce identical keys + assert_eq!(key1, key2); + } + + #[test] + fn test_global_cache_basic_operations() { + let cache = GlobalBlockCache::new(1, 4096); // 1MB, 4KB blocks + + let key = CacheKey::new(&PathBuf::from("test.sst"), 0); + let data = vec![1, 2, 3, 4, 5]; + + // Initially empty + assert!(cache.get(&key).is_none()); + + // Put and retrieve + cache.put(key.clone(), data.clone()); + let retrieved = cache.get(&key).unwrap(); + assert_eq!(*retrieved, data); + } + + #[test] + fn test_global_cache_arc_sharing() { + let cache = GlobalBlockCache::new(1, 4096); + + let key = CacheKey::new(&PathBuf::from("test.sst"), 0); + let data = vec![1, 2, 3, 4, 5]; + + cache.put(key.clone(), data.clone()); + + // Get twice and verify both point to same data + let ref1 = cache.get(&key).unwrap(); + let ref2 = cache.get(&key).unwrap(); + + // Arc should allow multiple references + assert_eq!(*ref1, *ref2); + assert_eq!(*ref1, data); + } + + #[test] + fn test_global_cache_capacity() { + let cache = GlobalBlockCache::new(1, 4096); // Can hold ~256 blocks (1MB / 4KB) + + let stats = cache.stats(); + assert_eq!(stats.len, 0); + assert!(stats.cap > 0); + assert_eq!(stats.cap, (1 * 1024 * 1024) / 4096); + } + + #[test] + fn test_global_cache_lru_eviction() { + // Small cache that can hold only 2 blocks + let cache = GlobalBlockCache::new(1, 512 * 1024); // ~2 blocks + + let key1 = CacheKey::new(&PathBuf::from("test1.sst"), 0); + let key2 = CacheKey::new(&PathBuf::from("test2.sst"), 0); + let key3 = CacheKey::new(&PathBuf::from("test3.sst"), 0); + + let data = vec![0u8; 1024]; // Small data + + cache.put(key1.clone(), data.clone()); + cache.put(key2.clone(), data.clone()); + + // Both should be in cache + assert!(cache.get(&key1).is_some()); + assert!(cache.get(&key2).is_some()); + + // Add third entry, should evict least recently used (key1) + cache.put(key3.clone(), data.clone()); + + let stats = cache.stats(); + assert!(stats.len <= stats.cap); + } + + #[test] + fn test_global_cache_clear() { + let cache = GlobalBlockCache::new(1, 4096); + + let key1 = CacheKey::new(&PathBuf::from("test1.sst"), 0); + let key2 = CacheKey::new(&PathBuf::from("test2.sst"), 0); + + cache.put(key1.clone(), vec![1, 2, 3]); + cache.put(key2.clone(), vec![4, 5, 6]); + + assert_eq!(cache.stats().len, 2); + + cache.clear(); + + assert_eq!(cache.stats().len, 0); + assert!(cache.get(&key1).is_none()); + assert!(cache.get(&key2).is_none()); + } + + #[test] + fn test_global_cache_update_existing_key() { + let cache = GlobalBlockCache::new(1, 4096); + + let key = CacheKey::new(&PathBuf::from("test.sst"), 0); + + cache.put(key.clone(), vec![1, 2, 3]); + let first = cache.get(&key).unwrap(); + assert_eq!(*first, vec![1, 2, 3]); + + // Update with new value + cache.put(key.clone(), vec![4, 5, 6]); + let second = cache.get(&key).unwrap(); + assert_eq!(*second, vec![4, 5, 6]); + + // Should still have only 1 entry + assert_eq!(cache.stats().len, 1); + } + + #[test] + fn test_global_cache_multiple_files_same_offset() { + let cache = GlobalBlockCache::new(1, 4096); + + let path1 = PathBuf::from("/data/file1.sst"); + let path2 = PathBuf::from("/data/file2.sst"); + + let key1 = CacheKey::new(&path1, 0); + let key2 = CacheKey::new(&path2, 0); + + cache.put(key1.clone(), vec![1, 1, 1]); + cache.put(key2.clone(), vec![2, 2, 2]); + + // Both should be retrievable independently + let data1 = cache.get(&key1).unwrap(); + let data2 = cache.get(&key2).unwrap(); + + assert_eq!(*data1, vec![1, 1, 1]); + assert_eq!(*data2, vec![2, 2, 2]); + + assert_eq!(cache.stats().len, 2); + } +} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 4107c64..c22a2e0 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,5 +1,6 @@ pub mod block; pub mod builder; +pub mod cache; pub mod config; pub mod reader; pub mod wal; diff --git a/src/storage/reader.rs b/src/storage/reader.rs index 0c83f62..7f21e6d 100644 --- a/src/storage/reader.rs +++ b/src/storage/reader.rs @@ -4,32 +4,41 @@ use crate::infra::config::StorageConfig; use crate::infra::error::{LsmError, Result}; use crate::storage::block::Block; use crate::storage::builder::{BlockMeta, MetaBlock}; +use crate::storage::cache::{CacheKey, GlobalBlockCache}; use bloomfilter::Bloom; -use lru::LruCache; use lz4_flex::decompress_size_prepended; use std::fs::File; use std::io::{Read, Seek, SeekFrom}; -use std::num::NonZeroUsize; use std::path::PathBuf; +use std::sync::Arc; const SST_MAGIC_V2: &[u8; 8] = b"LSMSST03"; const FOOTER_SIZE: u64 = 8; -/// SSTable V2 Reader with sparse index, Bloom filter, and block caching +/// SSTable V2 Reader with sparse index, Bloom filter, and shared global block caching #[derive(Debug)] pub struct SstableReader { metadata: MetaBlock, bloom_filter: Bloom<[u8]>, file: File, - block_cache: LruCache>, + block_cache: Arc, path: PathBuf, #[allow(dead_code)] config: StorageConfig, } impl SstableReader { - /// Open an SSTable V2 file for reading - pub fn open(path: PathBuf, config: StorageConfig) -> Result { + /// Open an SSTable V2 file for reading with a shared block cache + /// + /// # Arguments + /// * `path` - Path to the SSTable file + /// * `config` - Storage configuration + /// * `block_cache` - Shared global block cache + pub fn open( + path: PathBuf, + config: StorageConfig, + block_cache: Arc, + ) -> Result { let mut file = File::open(&path)?; // Verify magic number @@ -54,10 +63,6 @@ impl SstableReader { LsmError::CompactionFailed(format!("Bloom filter deserialization failed: {}", e)) })?; - // Initialize LRU cache - let cache_capacity = Self::calculate_cache_capacity(&config); - let block_cache = LruCache::new(cache_capacity); - Ok(Self { metadata, bloom_filter, @@ -234,16 +239,19 @@ impl SstableReader { } fn read_block(&mut self, block_meta: &BlockMeta) -> Result> { - // Check cache first - if let Some(cached) = self.block_cache.get(&block_meta.offset) { - return Ok(cached.clone()); + // Create cache key with file path and block offset + let cache_key = CacheKey::new(&self.path, block_meta.offset); + + // Check shared cache first + if let Some(cached) = self.block_cache.get(&cache_key) { + return Ok((*cached).clone()); } // Cache miss - read from disk let block_data = self.read_and_decompress_block(block_meta)?; - // Store in cache - self.block_cache.put(block_meta.offset, block_data.clone()); + // Store in shared cache + self.block_cache.put(cache_key, block_data.clone()); Ok(block_data) } @@ -301,13 +309,6 @@ impl SstableReader { // Return the block at idx - 1 (the last block where first_key <= search_key) Some(&self.metadata.blocks[idx - 1]) } - - fn calculate_cache_capacity(config: &StorageConfig) -> NonZeroUsize { - let cache_size_bytes = config.block_cache_size_mb * 1024 * 1024; - let avg_block_size = config.block_size; - let capacity = (cache_size_bytes / avg_block_size).max(1); - NonZeroUsize::new(capacity).unwrap_or(NonZeroUsize::new(100).unwrap()) - } } #[cfg(test)] @@ -320,11 +321,16 @@ mod tests { LogRecord::new(key.to_string(), value.to_vec()) } + fn create_test_cache(config: &StorageConfig) -> Arc { + GlobalBlockCache::new(config.block_cache_size_mb, config.block_size) + } + #[test] fn test_reader_basic_roundtrip() { let dir = tempdir().unwrap(); let path = dir.path().join("test.sst"); let config = StorageConfig::default(); + let cache = create_test_cache(&config); // Write SSTable let mut builder = SstableBuilder::new(path.clone(), config.clone(), 123).unwrap(); @@ -340,7 +346,7 @@ mod tests { builder.finish().unwrap(); // Read SSTable - let mut reader = SstableReader::open(path, config).unwrap(); + let mut reader = SstableReader::open(path, config, cache).unwrap(); // Verify reads let record1 = reader.get("key1").unwrap().unwrap(); @@ -361,6 +367,7 @@ mod tests { let dir = tempdir().unwrap(); let path = dir.path().join("bloom_test.sst"); let config = StorageConfig::default(); + let cache = create_test_cache(&config); // Write SSTable with known keys let mut builder = SstableBuilder::new(path.clone(), config.clone(), 456).unwrap(); @@ -373,7 +380,7 @@ mod tests { builder.finish().unwrap(); // Read and test Bloom filter - let reader = SstableReader::open(path, config).unwrap(); + let reader = SstableReader::open(path, config, cache).unwrap(); // Keys that exist should pass Bloom filter assert!(reader.might_contain("key_000")); @@ -399,6 +406,7 @@ mod tests { let path = dir.path().join("multi_block.sst"); let mut config = StorageConfig::default(); config.block_size = 256; // Small blocks to force multiple blocks + let cache = create_test_cache(&config); // Write many records to span multiple blocks let mut builder = SstableBuilder::new(path.clone(), config.clone(), 789).unwrap(); @@ -412,7 +420,7 @@ mod tests { builder.finish().unwrap(); // Read and verify all records - let mut reader = SstableReader::open(path, config).unwrap(); + let mut reader = SstableReader::open(path, config, cache).unwrap(); for i in 0..50 { let key = format!("key_{:03}", i); let record = reader.get(&key).unwrap(); @@ -425,6 +433,7 @@ mod tests { let dir = tempdir().unwrap(); let path = dir.path().join("boundary.sst"); let config = StorageConfig::default(); + let cache = create_test_cache(&config); // Write records with boundary keys let mut builder = SstableBuilder::new(path.clone(), config.clone(), 111).unwrap(); @@ -439,7 +448,7 @@ mod tests { .unwrap(); builder.finish().unwrap(); - let mut reader = SstableReader::open(path, config).unwrap(); + let mut reader = SstableReader::open(path, config, cache).unwrap(); // Test exact boundary keys assert!( @@ -483,6 +492,7 @@ mod tests { let dir = tempdir().unwrap(); let path = dir.path().join("scan_test.sst"); let config = StorageConfig::default(); + let cache = create_test_cache(&config); // Write ordered records let mut builder = SstableBuilder::new(path.clone(), config.clone(), 999).unwrap(); @@ -499,7 +509,7 @@ mod tests { builder.finish().unwrap(); // Scan all records - let mut reader = SstableReader::open(path, config).unwrap(); + let mut reader = SstableReader::open(path, config, cache).unwrap(); let records = reader.scan().unwrap(); assert_eq!(records.len(), test_keys.len(), "Should scan all records"); @@ -509,12 +519,13 @@ mod tests { fn test_reader_invalid_magic() { let dir = tempdir().unwrap(); let path = dir.path().join("invalid.sst"); + let config = StorageConfig::default(); + let cache = create_test_cache(&config); // Write file with wrong magic number std::fs::write(&path, b"INVALID_MAGIC").unwrap(); - let config = StorageConfig::default(); - let result = SstableReader::open(path, config); + let result = SstableReader::open(path, config, cache); assert!(result.is_err()); assert!(matches!( @@ -522,4 +533,48 @@ mod tests { LsmError::InvalidSstableFormat(_) )); } + + #[test] + fn test_shared_cache_across_readers() { + let dir = tempdir().unwrap(); + let config = StorageConfig::default(); + let cache = create_test_cache(&config); + + // Create two SSTable files + let path1 = dir.path().join("file1.sst"); + let path2 = dir.path().join("file2.sst"); + + // Write first SSTable + let mut builder1 = SstableBuilder::new(path1.clone(), config.clone(), 111).unwrap(); + builder1 + .add(b"key1", &create_test_record("key1", b"value1")) + .unwrap(); + builder1.finish().unwrap(); + + // Write second SSTable + let mut builder2 = SstableBuilder::new(path2.clone(), config.clone(), 222).unwrap(); + builder2 + .add(b"key2", &create_test_record("key2", b"value2")) + .unwrap(); + builder2.finish().unwrap(); + + // Open both readers with same cache + let mut reader1 = SstableReader::open(path1, config.clone(), Arc::clone(&cache)).unwrap(); + let mut reader2 = SstableReader::open(path2, config, Arc::clone(&cache)).unwrap(); + + let stats_before = cache.stats(); + + // Read from first SSTable (populates cache) + reader1.get("key1").unwrap(); + let stats_after1 = cache.stats(); + assert!(stats_after1.len >= stats_before.len); + + // Read from second SSTable (uses same cache) + reader2.get("key2").unwrap(); + let stats_after2 = cache.stats(); + assert!(stats_after2.len >= stats_after1.len); + + // Both readers share the same cache + assert!(stats_after2.len <= stats_after2.cap); + } } diff --git a/tests/integration_sstable_v2.rs b/tests/integration_sstable_v2.rs index 5a167bb..a96804e 100644 --- a/tests/integration_sstable_v2.rs +++ b/tests/integration_sstable_v2.rs @@ -2,18 +2,25 @@ use lsm_kv_store::core::log_record::LogRecord; use lsm_kv_store::infra::config::StorageConfig; use lsm_kv_store::infra::error::Result; use lsm_kv_store::storage::builder::SstableBuilder; +use lsm_kv_store::storage::cache::GlobalBlockCache; use lsm_kv_store::storage::reader::SstableReader; +use std::sync::Arc; use tempfile::tempdir; fn create_test_record(key: &str, value: &[u8]) -> LogRecord { LogRecord::new(key.to_string(), value.to_vec()) } +fn create_test_cache(config: &StorageConfig) -> Arc { + GlobalBlockCache::new(config.block_cache_size_mb, config.block_size) +} + #[test] fn test_sstable_v2_roundtrip_small() -> Result<()> { let dir = tempdir()?; let path = dir.path().join("roundtrip_small.sst"); let config = StorageConfig::default(); + let cache = create_test_cache(&config); // Write 10 records let mut builder = SstableBuilder::new(path.clone(), config.clone(), 123)?; @@ -27,7 +34,7 @@ fn test_sstable_v2_roundtrip_small() -> Result<()> { builder.finish()?; // Read and verify - let mut reader = SstableReader::open(path, config)?; + let mut reader = SstableReader::open(path, config, cache)?; for (key, expected_value) in &test_data { let record = reader.get(key)?.expect("Key should exist"); @@ -45,6 +52,7 @@ fn test_sstable_v2_roundtrip_large() -> Result<()> { let dir = tempdir()?; let path = dir.path().join("roundtrip_large.sst"); let config = StorageConfig::default(); + let cache = create_test_cache(&config); // Write 1000 records let mut builder = SstableBuilder::new(path.clone(), config.clone(), 456)?; @@ -58,7 +66,7 @@ fn test_sstable_v2_roundtrip_large() -> Result<()> { builder.finish()?; // Read and verify all records - let mut reader = SstableReader::open(path, config)?; + let mut reader = SstableReader::open(path, config, cache)?; for (key, expected_value) in &test_data { let record = reader.get(key)?.expect("Key should exist"); @@ -74,6 +82,7 @@ fn test_sstable_v2_multiple_blocks() -> Result<()> { let path = dir.path().join("multi_block.sst"); let mut config = StorageConfig::default(); config.block_size = 512; // Small blocks to force multiple blocks + let cache = create_test_cache(&config); // Write enough data to span multiple blocks let mut builder = SstableBuilder::new(path.clone(), config.clone(), 789)?; @@ -85,7 +94,7 @@ fn test_sstable_v2_multiple_blocks() -> Result<()> { builder.finish()?; // Read and verify - let mut reader = SstableReader::open(path, config)?; + let mut reader = SstableReader::open(path, config, cache)?; // Verify metadata shows multiple blocks assert!(reader.metadata().blocks.len() > 1, "Should have multiple blocks"); @@ -105,6 +114,7 @@ fn test_sstable_v2_bloom_filter_effectiveness() -> Result<()> { let dir = tempdir()?; let path = dir.path().join("bloom_test.sst"); let config = StorageConfig::default(); + let cache = create_test_cache(&config); // Write 500 records let mut builder = SstableBuilder::new(path.clone(), config.clone(), 999)?; @@ -115,7 +125,7 @@ fn test_sstable_v2_bloom_filter_effectiveness() -> Result<()> { builder.finish()?; // Test Bloom filter - let reader = SstableReader::open(path, config)?; + let reader = SstableReader::open(path, config, cache)?; // All existing keys should pass Bloom filter for i in 0..500 { @@ -139,6 +149,7 @@ fn test_sstable_v2_boundary_keys() -> Result<()> { let dir = tempdir()?; let path = dir.path().join("boundary.sst"); let config = StorageConfig::default(); + let cache = create_test_cache(&config); // Write records with boundary keys let mut builder = SstableBuilder::new(path.clone(), config.clone(), 111)?; @@ -147,7 +158,7 @@ fn test_sstable_v2_boundary_keys() -> Result<()> { builder.add(b"zzz", &create_test_record("zzz", b"last"))?; builder.finish()?; - let mut reader = SstableReader::open(path, config)?; + let mut reader = SstableReader::open(path, config, cache)?; // Test exact boundary keys assert!(reader.get("aaa")?.is_some(), "First key should exist"); @@ -172,6 +183,7 @@ fn test_sstable_v2_scan() -> Result<()> { let dir = tempdir()?; let path = dir.path().join("scan_test.sst"); let config = StorageConfig::default(); + let cache = create_test_cache(&config); // Write ordered records let mut builder = SstableBuilder::new(path.clone(), config.clone(), 222)?; @@ -183,7 +195,7 @@ fn test_sstable_v2_scan() -> Result<()> { builder.finish()?; // Scan all records - let mut reader = SstableReader::open(path, config)?; + let mut reader = SstableReader::open(path, config, cache)?; let records = reader.scan()?; assert_eq!(records.len(), test_keys.len(), "Should scan all records"); @@ -203,6 +215,7 @@ fn test_sstable_v2_large_values() -> Result<()> { let mut config = StorageConfig::default(); // Increase block size to accommodate large values config.block_size = 16384; // 16KB blocks + let cache = create_test_cache(&config); // Write records with large values (but smaller than block size) let mut builder = SstableBuilder::new(path.clone(), config.clone(), 333)?; @@ -215,7 +228,7 @@ fn test_sstable_v2_large_values() -> Result<()> { builder.finish()?; // Read and verify - let mut reader = SstableReader::open(path, config)?; + let mut reader = SstableReader::open(path, config, cache)?; for i in 0..10 { let key = format!("key_{}", i); @@ -234,6 +247,7 @@ fn test_sstable_v2_cache_effectiveness() -> Result<()> { let mut config = StorageConfig::default(); config.block_cache_size_mb = 10; // Small cache config.block_size = 512; + let cache = create_test_cache(&config); // Write multiple blocks let mut builder = SstableBuilder::new(path.clone(), config.clone(), 444)?; @@ -244,7 +258,7 @@ fn test_sstable_v2_cache_effectiveness() -> Result<()> { } builder.finish()?; - let mut reader = SstableReader::open(path, config)?; + let mut reader = SstableReader::open(path, config, cache)?; // Read same keys multiple times (should benefit from cache) for _ in 0..3 { @@ -263,6 +277,7 @@ fn test_sstable_v2_empty_key() -> Result<()> { let dir = tempdir()?; let path = dir.path().join("empty_key.sst"); let config = StorageConfig::default(); + let cache = create_test_cache(&config); // Write with empty string key let mut builder = SstableBuilder::new(path.clone(), config.clone(), 555)?; @@ -270,7 +285,7 @@ fn test_sstable_v2_empty_key() -> Result<()> { builder.add(b"normal_key", &create_test_record("normal_key", b"normal_value"))?; builder.finish()?; - let mut reader = SstableReader::open(path, config)?; + let mut reader = SstableReader::open(path, config, cache)?; // Should be able to read empty key let record = reader.get("")?.expect("Empty key should exist"); @@ -288,6 +303,7 @@ fn test_sstable_v2_unicode_keys() -> Result<()> { let dir = tempdir()?; let path = dir.path().join("unicode.sst"); let config = StorageConfig::default(); + let cache = create_test_cache(&config); // Write with unicode keys (pre-sorted by UTF-8 byte order) let mut builder = SstableBuilder::new(path.clone(), config.clone(), 666)?; @@ -300,7 +316,7 @@ fn test_sstable_v2_unicode_keys() -> Result<()> { } builder.finish()?; - let mut reader = SstableReader::open(path, config)?; + let mut reader = SstableReader::open(path, config, cache)?; // Verify all unicode keys are readable for key in &unicode_keys {