diff --git a/bannet/connection.go b/bannet/connection.go index 9767405..5307f34 100644 --- a/bannet/connection.go +++ b/bannet/connection.go @@ -31,6 +31,10 @@ type Connection struct { property map[string]any propertyLock sync.RWMutex + + // 构造时从全局配置快照的两项策略,避免在每帧读取路径上访问可变全局状态。 + maxPackageSize uint32 + useWorkerPool bool } func NewConnection(conn *net.TCPConn, connID uint32, handle Dispatcher, server *Server) *Connection { @@ -45,6 +49,9 @@ func NewConnection(conn *net.TCPConn, connID uint32, handle Dispatcher, server * msgChan: make(chan []byte, 10), // 高优通道加小缓冲,避免硬阻塞 msgBuffChan: make(chan []byte, config.G.MaxMsgChanLen), property: make(map[string]any), // 必须初始化,否则 SetProperty 写 nil map 会 panic + + maxPackageSize: config.G.MaxPackageSize, + useWorkerPool: config.G.WorkerPoolSize > 0, } c.TCPServer.Conns().Add(c) return c @@ -75,6 +82,12 @@ func (c *Connection) StartReader() { slog.Error("conn unpack header failed", "connID", c.ConnID, "error", err) return } + // 帧长上限在此执行:读取负载之前拒绝超限帧,避免按对端声称的长度分配内存。 + if c.maxPackageSize > 0 && msg.MsgLen() > c.maxPackageSize { + slog.Warn("conn frame exceeds max package size", + "connID", c.ConnID, "dataLen", msg.MsgLen(), "max", c.maxPackageSize) + return + } // 头部之后, 先按 IDLen 读取 msgID 字符串 if msg.IDLen > 0 { @@ -97,8 +110,8 @@ func (c *Connection) StartReader() { } msg.SetData(data) req := newRequest(msg, c) - // 根据有没有启动 WorkPool 选择不同的结果 - if config.G.WorkerPoolSize > 0 { + // 根据有没有启动 worker 池选择投递方式 + if c.useWorkerPool { c.MsgHandle.SendMsgToTaskQueue(req) } else { go c.MsgHandle.DoMsgHandle(req) diff --git a/bannet/datapack.go b/bannet/datapack.go index 8e4625b..5fd56f7 100644 --- a/bannet/datapack.go +++ b/bannet/datapack.go @@ -4,9 +4,6 @@ import ( "encoding/binary" "errors" "fmt" - "log/slog" - - "github.com/NeverENG/BanDB/config" ) // 报文格式: @@ -41,7 +38,13 @@ func (dp *DataPack) Pack(msg *Message) ([]byte, error) { return buf, nil } -// UnPack 只解析定长头部 (6 字节), 返回带 DataLen 与 IDLen 的占位 Message; +// UnPack 只解析定长头部。 +// +// 它不校验帧长上限:那是策略而非编解码,由连接侧在读取负载前执行(见 Connection. +// StartReader)。此前该校验在此处每帧读两次全局配置——把策略留在边界,解码器才能保持 +// 无状态、不依赖全局。 +// +// 原注释:只解析定长头部 (6 字节), 返回带 DataLen 与 IDLen 的占位 Message; // 调用方拿到 IDLen 后, 还需要从连接读取 IDLen+DataLen 字节填充 Id 与 Data。 func (dp *DataPack) UnPack(data []byte) (*Message, error) { if len(data) < int(dp.HeadLen()) { @@ -53,10 +56,6 @@ func (dp *DataPack) UnPack(data []byte) (*Message, error) { msg.DataLen = binary.LittleEndian.Uint32(data[0:4]) idLen := binary.LittleEndian.Uint16(data[4:6]) - if config.G.MaxPackageSize > 0 && msg.DataLen > config.G.MaxPackageSize { - slog.Warn("banNet frame exceeds max package size", "dataLen", msg.DataLen, "max", config.G.MaxPackageSize) - return nil, errors.New("data too large") - } // 借用 Id 暂存 IDLen 信息: 调用方先从 MsgID() 拿不到东西, 通过头部之后另读 IDLen 字节填回。 // 这里用 SetMsgLen 仅保留 DataLen 不冲突, IDLen 通过返回的 Message.IDLen 提供。 msg.IDLen = idLen diff --git a/bannet/oversized_frame_test.go b/bannet/oversized_frame_test.go new file mode 100644 index 0000000..8c44422 --- /dev/null +++ b/bannet/oversized_frame_test.go @@ -0,0 +1,91 @@ +package bannet_test + +import ( + "encoding/binary" + "net" + "strconv" + "testing" + "time" + + "github.com/NeverENG/BanDB/bannet" + "github.com/NeverENG/BanDB/config" + "github.com/NeverENG/BanDB/proto" +) + +// countingHandler 记录被分派到的请求数,用于断言超限帧未进入业务处理。 +type countingHandler struct { + bannet.BaseRouter + handled chan struct{} +} + +func (h *countingHandler) Handle(bannet.Request) { + select { + case h.handled <- struct{}{}: + default: + } +} + +// TestOversizedFrameRejectedBeforeReadingPayload 守护帧长上限的执行点。 +// +// 该校验此前在 DataPack.UnPack 内,每帧读两次全局配置;现已移到连接侧(读取负载之前), +// 让编解码器保持无状态。移动执行点意味着必须有测试证明它仍在执行——否则一个恶意或损坏的 +// 帧头就能让服务端按对端声称的长度分配内存。 +// +// 构造方式:只发一个声称负载极大的帧头,不发负载。服务端必须在读取负载前就断开连接, +// 且该帧不得被分派给业务处理。 +func TestOversizedFrameRejectedBeforeReadingPayload(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + host, portStr, _ := net.SplitHostPort(ln.Addr().String()) + port, _ := strconv.Atoi(portStr) + addr := ln.Addr().String() + ln.Close() + + oldMax := config.G.MaxPackageSize + config.G.MaxPackageSize = 1024 + t.Cleanup(func() { config.G.MaxPackageSize = oldMax }) + + h := &countingHandler{handled: make(chan struct{}, 1)} + srv := bannet.NewServer() + srv.IP, srv.Port = host, port + srv.AddRouter(proto.MsgPut, h) + srv.Start() + t.Cleanup(srv.Stop) + + var conn net.Conn + for i := 0; i < 100; i++ { + if conn, err = net.DialTimeout("tcp", addr, 200*time.Millisecond); err == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + if conn == nil { + t.Fatalf("服务端未就绪: %v", err) + } + defer conn.Close() + + // 帧头: [dataLen u32 LE][idLen u16 LE],声称 64MiB 负载但一个字节都不发。 + head := make([]byte, 6) + binary.LittleEndian.PutUint32(head[0:4], 64<<20) + binary.LittleEndian.PutUint16(head[4:6], uint16(len(proto.MsgPut))) + if _, err := conn.Write(append(head, []byte(proto.MsgPut)...)); err != nil { + t.Fatalf("write: %v", err) + } + + // 服务端应断开连接:读将返回 EOF/RST,而不是一直等着那 64MiB。 + conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + buf := make([]byte, 1) + if _, err := conn.Read(buf); err == nil { + t.Fatal("超限帧后连接仍可读,应已被服务端关闭") + } else if ne, ok := err.(net.Error); ok && ne.Timeout() { + t.Fatal("服务端既未拒绝也未关闭连接——很可能正按声称的长度等待/分配 64MiB") + } + + select { + case <-h.handled: + t.Fatal("超限帧不应被分派到业务处理") + default: + } +} diff --git a/cmd/ban-ingest/main.go b/cmd/ban-ingest/main.go index 96cb7b9..a26ff61 100644 --- a/cmd/ban-ingest/main.go +++ b/cmd/ban-ingest/main.go @@ -99,7 +99,7 @@ func setupEngine(memTableSize int) (*storage.Engine, func()) { config.G.WALPath = filepath.Join(tmp, "wal.log") config.G.MaxMemTableSize = memTableSize - memTable := storage.NewEngine() + memTable := storage.NewEngine(storage.DefaultOptions()) cleanup := func() { _ = memTable.Close() os.RemoveAll(tmp) diff --git a/service/fsm.go b/service/fsm.go index 387ddda..9e10b90 100644 --- a/service/fsm.go +++ b/service/fsm.go @@ -52,7 +52,7 @@ type KVServer struct { func NewKVServer() *KVServer { // 初始化存储 kv := &KVServer{ - storage: storage.NewEngine(), + storage: storage.NewEngine(storage.DefaultOptions()), } if config.G.Mode == config.ModeStandalone { diff --git a/storage/bench_test.go b/storage/bench_test.go index 39db454..99bea99 100644 --- a/storage/bench_test.go +++ b/storage/bench_test.go @@ -2,15 +2,12 @@ package storage_test import ( "fmt" - "testing" - - "github.com/NeverENG/BanDB/config" "github.com/NeverENG/BanDB/storage" + "testing" ) func benchmarkEnginePut(b *testing.B, valueSize int) { - config.G.MaxMemTableSize = 1000000 // prevent flush during bench - memTable := storage.NewEngine() + memTable := storage.NewEngine(storage.Options{Dir: b.TempDir(), MaxMemTableSize: 1000000}) // 阈值取大,基准期间不触发 flush value := make([]byte, valueSize) for i := range value { @@ -30,8 +27,7 @@ func BenchmarkEngine_Put_1KB(b *testing.B) { benchmarkEnginePut(b, 1024) } func BenchmarkEngine_Put_4KB(b *testing.B) { benchmarkEnginePut(b, 4096) } func BenchmarkEngine_Get(b *testing.B) { - config.G.MaxMemTableSize = 1000000 - memTable := storage.NewEngine() + memTable := storage.NewEngine(storage.Options{Dir: b.TempDir(), MaxMemTableSize: 1000000}) // 阈值取大,基准期间不触发 flush value := make([]byte, 256) for i := 0; i < 10000; i++ { @@ -47,8 +43,7 @@ func BenchmarkEngine_Get(b *testing.B) { } func BenchmarkEngine_Delete(b *testing.B) { - config.G.MaxMemTableSize = 1000000 - memTable := storage.NewEngine() + memTable := storage.NewEngine(storage.Options{Dir: b.TempDir(), MaxMemTableSize: 1000000}) // 阈值取大,基准期间不触发 flush value := make([]byte, 256) keys := make([][]byte, b.N) @@ -64,8 +59,7 @@ func BenchmarkEngine_Delete(b *testing.B) { } func BenchmarkMemTable_Put(b *testing.B) { - config.G.MaxMemTableSize = 1000000 - mt := storage.NewEngine() + mt := storage.NewEngine(storage.Options{Dir: b.TempDir(), MaxMemTableSize: 1000000}) // 阈值取大,基准期间不触发 flush value := []byte("benchmark-value-data-256-bytes-padding-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx") b.ResetTimer() @@ -76,8 +70,7 @@ func BenchmarkMemTable_Put(b *testing.B) { } func BenchmarkMemTable_Get(b *testing.B) { - config.G.MaxMemTableSize = 1000000 - mt := storage.NewEngine() + mt := storage.NewEngine(storage.Options{Dir: b.TempDir(), MaxMemTableSize: 1000000}) // 阈值取大,基准期间不触发 flush value := []byte("benchmark-value") for i := 0; i < 100000; i++ { key := []byte(fmt.Sprintf("key-%08d", i)) diff --git a/storage/compaction_bench_test.go b/storage/compaction_bench_test.go index 184df2e..9ff2cbf 100644 --- a/storage/compaction_bench_test.go +++ b/storage/compaction_bench_test.go @@ -5,8 +5,6 @@ import ( "runtime" "testing" "time" - - "github.com/NeverENG/BanDB/config" ) // TestCompactionBench 是 compaction 压测台:确定性地驱动真实的 flush + compaction 级联, @@ -17,16 +15,13 @@ import ( func TestCompactionBench(t *testing.T) { // 小 memtable + 小 compaction 阈值,逼出频繁 flush 与多级 compaction。 dir := t.TempDir() - oldPath, oldComp := config.G.SSTablePath, config.G.MaxCompactionSize - config.G.SSTablePath = dir - config.G.MaxCompactionSize = 4 - defer func() { config.G.SSTablePath, config.G.MaxCompactionSize = oldPath, oldComp }() + opts := Options{Dir: dir, MaxCompactionSize: 4} ResetCompactionStats() // 不经 NewEngine(避免启动异步 FlushWorker/ListenCompactCh 造成非确定性), // 只用 sst,手动驱动 flush 与 compaction。 - mt := newBareMemTable(NewSSTable()) + mt := newBareMemTable(NewSSTable(opts), opts) const ( flushes = 300 @@ -90,7 +85,7 @@ func TestCompactionBench(t *testing.T) { } // === 模拟重启:LoadSSTableMetaList 把所有文件 Level 归 0(level 未持久化)=== - sst2 := NewSSTable() + sst2 := NewSSTable(opts) sst2.LoadSSTableMetaList() time.Sleep(150 * time.Millisecond) // 等异步预热 goroutine 落定,避免与后续 compaction 竞争 dist2, total2 := levelDistribution(sst2) @@ -101,7 +96,7 @@ func TestCompactionBench(t *testing.T) { } t.Logf("文件总数=%d per-level=%s ← %s", total2, dist2, collapsed) - mt2 := newBareMemTable(sst2) + mt2 := newBareMemTable(sst2, opts) before := ReadCompactionStats() t0 := time.Now() mt2.CompactSSTable(0) diff --git a/storage/compaction_failure_test.go b/storage/compaction_failure_test.go index 13573ed..dda2598 100644 --- a/storage/compaction_failure_test.go +++ b/storage/compaction_failure_test.go @@ -4,8 +4,6 @@ import ( "fmt" "os" "testing" - - "github.com/NeverENG/BanDB/config" ) // writeSSTables 写出 n 个各含若干 key 的 SSTable,返回它们的路径。 @@ -39,11 +37,9 @@ func writeSSTables(t *testing.T, ss *SSTable, n, perFile int) []string { // 故 root 下会失效——检测到仍能创建文件时直接跳过,而不是给出假绿。 func TestCompactionFailureKeepsSourceFiles(t *testing.T) { dir := t.TempDir() - oldSST := config.G.SSTablePath - config.G.SSTablePath = dir - t.Cleanup(func() { config.G.SSTablePath = oldSST }) + opts := Options{Dir: dir} - ss := NewSSTable() + ss := NewSSTable(opts) paths := writeSSTables(t, ss, 3, 50) if err := os.Chmod(dir, 0o500); err != nil { // r-x:可读可遍历,不可创建新文件 @@ -87,14 +83,9 @@ func TestCompactionFailureKeepsSourceFiles(t *testing.T) { // 在合并失败时不得删除任何源文件,且全部 key 仍可读。 func TestCompactSSTableKeepsSourcesOnMergeFailure(t *testing.T) { dir := t.TempDir() - oldSST, oldCompact := config.G.SSTablePath, config.G.MaxCompactionSize - config.G.SSTablePath = dir - config.G.MaxCompactionSize = 2 // 两个文件即触发合并 - t.Cleanup(func() { - config.G.SSTablePath, config.G.MaxCompactionSize = oldSST, oldCompact - }) + opts := Options{Dir: dir, MaxCompactionSize: 2} // 两个文件即触发合并 - e := NewEngine() + e := NewEngine(opts) t.Cleanup(func() { e.Close() }) paths := writeSSTables(t, e.sst, 3, 30) diff --git a/storage/compaction_level_test.go b/storage/compaction_level_test.go index 2e9a2fc..13fe66a 100644 --- a/storage/compaction_level_test.go +++ b/storage/compaction_level_test.go @@ -4,8 +4,6 @@ import ( "fmt" "testing" "time" - - "github.com/NeverENG/BanDB/config" ) // TestCompaction_LevelPersistedAcrossRestart 是「重启不塌缩」的回归守卫: @@ -13,12 +11,9 @@ import ( // 断言 per-level 分布被保留,而非全部塌缩到 L0(后者会在重启后触发全量重写)。 func TestCompaction_LevelPersistedAcrossRestart(t *testing.T) { dir := t.TempDir() - oldPath, oldComp := config.G.SSTablePath, config.G.MaxCompactionSize - config.G.SSTablePath = dir - config.G.MaxCompactionSize = 4 - defer func() { config.G.SSTablePath, config.G.MaxCompactionSize = oldPath, oldComp }() + opts := Options{Dir: dir, MaxCompactionSize: 4} - mt := newBareMemTable(NewSSTable()) + mt := newBareMemTable(NewSSTable(opts), opts) val := make([]byte, 32) global := 0 for f := 0; f < 40; f++ { @@ -40,7 +35,7 @@ func TestCompaction_LevelPersistedAcrossRestart(t *testing.T) { } // 模拟重启:全新 SSTable 从磁盘恢复。 - sst2 := NewSSTable() + sst2 := NewSSTable(opts) sst2.LoadSSTableMetaList() time.Sleep(150 * time.Millisecond) after, totalAfter := levelDistribution(sst2) diff --git a/storage/engine.go b/storage/engine.go index 00caa78..6425d4c 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -6,16 +6,10 @@ import ( "log/slog" "sync" - "github.com/NeverENG/BanDB/config" "github.com/NeverENG/BanDB/internal/credit" "github.com/NeverENG/BanDB/internal/metrics" ) -var ( - maxLevel = config.G.MaxMemTableLevel - probability = config.G.MaxMemTableP -) - // Engine 是 LSM 存储引擎:它同时持有内存中的表与磁盘上的 SSTable 集合,并驱动二者 // 之间的流转。 // @@ -51,22 +45,29 @@ type Engine struct { // 并发修改全局配置形成数据竞争。 maxSize int maxCompaction int + + // opts 是构造时传入的参数,引擎此后只认它,不再读全局配置。 + opts Options } // SkipNode 跳表节点 -func NewEngine() *Engine { +// NewEngine 按 opts 构造存储引擎。参数在此固定,此后不再读全局配置。 +// 需要沿用进程配置时传 DefaultOptions()。 +func NewEngine(opts Options) *Engine { + opts = opts.withDefaults() mt := &Engine{ - active: newSkipList(), + active: newSkipList(opts.SkipListMaxLevel, opts.SkipListP), flushCh: make(chan struct{}, 1), compactCh: make(chan struct{}, 1), stopCh: make(chan struct{}), - sst: NewSSTable(), - credits: credit.New(config.G.MemTableMaxInflightBytes), - maxSize: config.G.MaxMemTableSize, - maxCompaction: config.G.MaxCompactionSize, + sst: NewSSTable(opts), + credits: credit.New(opts.MaxInflightBytes), + maxSize: opts.MaxMemTableSize, + maxCompaction: opts.MaxCompactionSize, + opts: opts, } // 注册未 flush 字节数仪表,供周期性指标快照实时读取。 - metrics.SetMemTableGauges(mt.InflightBytes, config.G.MemTableMaxInflightBytes) + metrics.SetMemTableGauges(mt.InflightBytes, opts.MaxInflightBytes) go mt.FlushWorker() go mt.ListenCompactCh() @@ -314,7 +315,7 @@ func (m *Engine) Flush() { return } m.dirty = m.active - m.active = newSkipList() + m.active = newSkipList(m.opts.SkipListMaxLevel, m.opts.SkipListP) } dirty := m.dirty m.mu.Unlock() @@ -393,7 +394,7 @@ func (m *Engine) FlushToSSTable(entries []LogEntry) error { // 创建临时跳表,按序插入(同 key 自动去重/更新)。 // Value==nil 为墓碑,按墓碑插入而非物理删除:快照中的删除需写入墓碑以 shadow // 旧 SSTable 中的同名值,否则该 key 会在读路径被「复活」。 - tmp := newSkipList() + tmp := newSkipList(m.opts.SkipListMaxLevel, m.opts.SkipListP) for _, entry := range entries { tmp.insert(entry.Key, entry.Value) } diff --git a/storage/engine_semantics_test.go b/storage/engine_semantics_test.go index fc3ecbb..2e1a1bf 100644 --- a/storage/engine_semantics_test.go +++ b/storage/engine_semantics_test.go @@ -1,23 +1,11 @@ package storage import ( - "github.com/NeverENG/BanDB/config" - "os" "testing" ) func TestMemTable_PutAndDelete(t *testing.T) { - // 配置临时 WAL - oldWALPath := config.G.WALPath - testWAL := "test_memtable_wal.log" - config.G.WALPath = testWAL - os.Remove(testWAL) // 先删除旧的 - defer func() { - os.Remove(testWAL) - config.G.WALPath = oldWALPath - }() - - memTable := NewEngine() + memTable := NewEngine(testOptions(t)) t.Log("MemTable created") err := memTable.Put([]byte("key1"), []byte("value1")) @@ -45,28 +33,17 @@ func TestMemTable_PutAndDelete(t *testing.T) { } // setupMemTableTempEnv 为 MemTable 测试配置隔离的 WAL/SSTable 路径并关闭自动 flush。 -func setupMemTableTempEnv(t *testing.T, walName string) { +func setupMemTableTempEnv(t *testing.T) Options { t.Helper() - oldWAL := config.G.WALPath - oldSST := config.G.SSTablePath - oldMax := config.G.MaxMemTableSize - config.G.WALPath = walName - config.G.SSTablePath = t.TempDir() - config.G.MaxMemTableSize = 1 << 20 - os.Remove(walName) - t.Cleanup(func() { - os.Remove(walName) - config.G.WALPath = oldWAL - config.G.SSTablePath = oldSST - config.G.MaxMemTableSize = oldMax - }) + // 阈值取大,避免 Put 触发自动 flush——这些用例要自己控制 flush 时机。 + return Options{Dir: t.TempDir(), MaxMemTableSize: 1 << 20} } // TestMemTableDeleteFlushedKeyNoResurrect 删除一个已 flush 到 SSTable 的 key: // 墓碑必须在 active 与落盘后都 shadow 旧值,且后续 Put 可复活该 key。 func TestMemTableDeleteFlushedKeyNoResurrect(t *testing.T) { - setupMemTableTempEnv(t, "test_tombstone_wal.log") - m := NewEngine() + opts := setupMemTableTempEnv(t) + m := NewEngine(opts) if err := m.Put([]byte("k"), []byte("v")); err != nil { t.Fatalf("put: %v", err) @@ -98,8 +75,8 @@ func TestMemTableDeleteFlushedKeyNoResurrect(t *testing.T) { // TestMemTableEmptyValueNotTombstone 空值是真实值, 不能被当作墓碑: // Put(k, []byte{}) 经 flush 落盘后, Get 必须返回 found+空, 而非未找到。 func TestMemTableEmptyValueNotTombstone(t *testing.T) { - setupMemTableTempEnv(t, "test_emptyval_wal.log") - m := NewEngine() + opts := setupMemTableTempEnv(t) + m := NewEngine(opts) if err := m.Put([]byte("e"), []byte{}); err != nil { t.Fatalf("put empty: %v", err) @@ -122,21 +99,7 @@ func TestMemTableEmptyValueNotTombstone(t *testing.T) { // SSTable 中,Get 必须返回最新版本。当前读路径按文件最旧在前取首个命中,会返回 // 陈旧值——本测试即该正确性 bug 的回归门。 func TestGetReturnsNewestAcrossSSTables(t *testing.T) { - oldWAL := config.G.WALPath - oldSST := config.G.SSTablePath - oldMax := config.G.MaxMemTableSize - config.G.WALPath = "test_newest_wal.log" - config.G.SSTablePath = t.TempDir() - config.G.MaxMemTableSize = 1 << 20 // 避免 Put 触发自动 flush,由测试显式 Flush 控制 - os.Remove(config.G.WALPath) - defer func() { - os.Remove(config.G.WALPath) - config.G.WALPath = oldWAL - config.G.SSTablePath = oldSST - config.G.MaxMemTableSize = oldMax - }() - - m := NewEngine() + m := NewEngine(setupMemTableTempEnv(t)) if err := m.Put([]byte("k"), []byte("v1")); err != nil { t.Fatalf("put v1: %v", err) diff --git a/storage/engine_test.go b/storage/engine_test.go index 3fa0c0e..604a831 100644 --- a/storage/engine_test.go +++ b/storage/engine_test.go @@ -1,39 +1,13 @@ package storage import ( - "path/filepath" "testing" - - "github.com/NeverENG/BanDB/config" ) func setupTestEngine(t *testing.T) (*Engine, func()) { - oldWALPath := config.G.WALPath - oldMaxSize := config.G.MaxMemTableSize - oldSSTPath := config.G.SSTablePath - - // 每个用例独立的临时目录,避免读到共享 ../../log 下其它运行的残留 .sst/WAL - dir := t.TempDir() - config.G.WALPath = filepath.Join(dir, "wal.log") - config.G.SSTablePath = dir - config.G.MaxMemTableSize = 100 - - memTable := NewEngine() - - // 不再另起 FlushWorker:NewEngine 已启动一个。两个 worker 会并发进入 Flush, - // 各自取到不同的 dirty 表后互相把 m.dirty 置 nil,导致 flush 丢数据、读回缺失 - // (表现为 TestEngine_* 偶发失败)。 - - cleanup := func() { - // 关闭 WAL 文件(临时目录由 t.TempDir 自动清理) - memTable.Close() - // 恢复配置 - config.G.WALPath = oldWALPath - config.G.SSTablePath = oldSSTPath - config.G.MaxMemTableSize = oldMaxSize - } - - return memTable, cleanup + // 每个用例独立的临时目录与独立的参数,用例之间不再经由全局配置互相影响。 + mt := NewEngine(Options{Dir: t.TempDir(), MaxMemTableSize: 100}) + return mt, func() { mt.Close() } } func TestEngine_PutAndGet(t *testing.T) { @@ -161,15 +135,7 @@ func TestEngine_UpdateExistingKey(t *testing.T) { } func TestEngine_PutTriggersFlush(t *testing.T) { - oldMaxSize := config.G.MaxMemTableSize - oldWALPath := config.G.WALPath - oldSSTPath := config.G.SSTablePath - config.G.MaxMemTableSize = 5 - dir := t.TempDir() - config.G.WALPath = filepath.Join(dir, "wal.log") - config.G.SSTablePath = dir - - mt := NewEngine() + mt := NewEngine(Options{Dir: t.TempDir(), MaxMemTableSize: 5}) for i := 0; i < 10; i++ { key := []byte(string(rune('a' + i))) @@ -185,7 +151,4 @@ func TestEngine_PutTriggersFlush(t *testing.T) { } mt.Close() - config.G.MaxMemTableSize = oldMaxSize - config.G.WALPath = oldWALPath - config.G.SSTablePath = oldSSTPath } diff --git a/storage/errors_test.go b/storage/errors_test.go index cbc3d18..f448762 100644 --- a/storage/errors_test.go +++ b/storage/errors_test.go @@ -4,20 +4,15 @@ import ( "errors" "path/filepath" "testing" - - "github.com/NeverENG/BanDB/config" ) // TestGetMissingKeyIsErrKeyNotFound 固定「key 不存在」的错误契约:调用方须能以 // errors.Is 判别,从而与读盘失败等真实故障区分。此前该错误在四处各自 errors.New // 一个新对象,任何调用方都无法判别,只能一律当作失败处理。 func TestGetMissingKeyIsErrKeyNotFound(t *testing.T) { - dir := t.TempDir() - oldSST := config.G.SSTablePath - config.G.SSTablePath = dir - t.Cleanup(func() { config.G.SSTablePath = oldSST }) + opts := testOptions(t) - mt := NewEngine() + mt := NewEngine(opts) t.Cleanup(func() { mt.Close() }) if err := mt.Put([]byte("present"), []byte("v")); err != nil { @@ -43,11 +38,9 @@ func TestGetMissingKeyIsErrKeyNotFound(t *testing.T) { // TestWriteEmptySSTableIsErrNoEntries 固定空条目集落盘的错误契约。 func TestWriteEmptySSTableIsErrNoEntries(t *testing.T) { - oldSST := config.G.SSTablePath - config.G.SSTablePath = filepath.Join(t.TempDir(), "sst") - t.Cleanup(func() { config.G.SSTablePath = oldSST }) + opts := Options{Dir: filepath.Join(t.TempDir(), "sst")} - ss := NewSSTable() + ss := NewSSTable(opts) if err := ss.WriteToSSTable(nil); !errors.Is(err, ErrNoEntries) { t.Fatalf("写入空条目集应为 ErrNoEntries, 实际: %v", err) } diff --git a/storage/merge_test.go b/storage/merge_test.go index e60db20..82e70c3 100644 --- a/storage/merge_test.go +++ b/storage/merge_test.go @@ -3,25 +3,21 @@ package storage import ( "path/filepath" "testing" - - "github.com/NeverENG/BanDB/config" ) func entry(k, v string) LogEntry { return LogEntry{Key: []byte(k), Value: []byte(v)} } -func withTempSSTDir(t *testing.T) { +func withTempSSTDir(t *testing.T) Options { t.Helper() - old := config.G.SSTablePath - config.G.SSTablePath = t.TempDir() - t.Cleanup(func() { config.G.SSTablePath = old }) + return Options{Dir: t.TempDir()} } // TestSSTableIteratorStopsAtDataEnd 迭代器只读数据区, 不把块索引/布隆当作条目。 func TestSSTableIteratorStopsAtDataEnd(t *testing.T) { - withTempSSTDir(t) - ss := NewSSTable() + opts := withTempSSTDir(t) + ss := NewSSTable(opts) if err := ss.WriteToSSTable([]LogEntry{ entry("k1", "v1"), entry("k2", "v2"), entry("k3", "v3"), }); err != nil { @@ -55,8 +51,8 @@ func TestSSTableIteratorStopsAtDataEnd(t *testing.T) { // TestMergeBasic 多个不相交源合并: 全部 key 有序保留、可经现有读路径读取、布隆可用。 func TestMergeBasic(t *testing.T) { - withTempSSTDir(t) - ss := NewSSTable() + opts := withTempSSTDir(t) + ss := NewSSTable(opts) if err := ss.WriteToSSTable([]LogEntry{entry("a", "1"), entry("c", "1"), entry("e", "1")}); err != nil { t.Fatal(err) } @@ -95,7 +91,7 @@ func TestMergeBasic(t *testing.T) { t.Errorf("read d: ok=%v v=%q", ok, v) } // 布隆已写入且能否决缺失 key - if NewSSTable().getBloom(merged.Filepath) == nil { + if NewSSTable(opts).getBloom(merged.Filepath) == nil { t.Error("merged file should carry a bloom filter") } if _, ok := ss.ReadFromSSTable(merged.Filepath, []byte("zzz")); ok { @@ -105,8 +101,8 @@ func TestMergeBasic(t *testing.T) { // TestMergeDedupKeepNewest 同 key 出现在多个源时, srcIdx 最大(最新)者胜出且只出现一次。 func TestMergeDedupKeepNewest(t *testing.T) { - withTempSSTDir(t) - ss := NewSSTable() + opts := withTempSSTDir(t) + ss := NewSSTable(opts) // 三个文件按加入顺序 srcIdx = 0,1,2; "dup" 的最新值应为 v2 ss.WriteToSSTable([]LogEntry{entry("a", "a0"), entry("dup", "v0")}) ss.WriteToSSTable([]LogEntry{entry("dup", "v1"), entry("m", "m1")}) @@ -145,12 +141,12 @@ func TestMergeDedupKeepNewest(t *testing.T) { // TestMergeRejectsUnsortedSource 源非升序时归并应失败(返回 nil), 而非静默产出错误结果。 func TestMergeRejectsUnsortedSource(t *testing.T) { - withTempSSTDir(t) - path := filepath.Join(config.G.SSTablePath, "bad.sst") + opts := withTempSSTDir(t) + path := filepath.Join(opts.Dir, "bad.sst") // 降序 key — 违反归并前提 writeV1SSTable(t, path, []LogEntry{entry("c", "1"), entry("b", "1"), entry("a", "1")}) - ss := NewSSTable() + ss := NewSSTable(opts) merged := ss.MergeSSTable([]*SSTableMeta{{Filepath: path}}, 1) if merged != nil { t.Error("merge should fail on a non-ascending source, got non-nil") @@ -163,8 +159,8 @@ func tomb(k string) LogEntry { return LogEntry{Key: []byte(k), Value: nil} } // TestSSTableTombstoneRoundTrip 墓碑经 SSTable 写→点查→ReadAll→迭代器全链路: // 哨兵长度不触发巨型分配,墓碑还原为 found+nil,真实值不受影响。 func TestSSTableTombstoneRoundTrip(t *testing.T) { - withTempSSTDir(t) - ss := NewSSTable() + opts := withTempSSTDir(t) + ss := NewSSTable(opts) if err := ss.WriteToSSTable([]LogEntry{ entry("a", "1"), tomb("del"), entry("z", "3"), }); err != nil { @@ -207,8 +203,8 @@ func TestSSTableTombstoneRoundTrip(t *testing.T) { // TestMergePreservesTombstone 旧文件含值、新文件含同 key 墓碑:归并须保留墓碑 // (newest 胜出),compaction 后点查仍为已删除,而不是把旧值复活。 func TestMergePreservesTombstone(t *testing.T) { - withTempSSTDir(t) - ss := NewSSTable() + opts := withTempSSTDir(t) + ss := NewSSTable(opts) ss.WriteToSSTable([]LogEntry{entry("k", "v")}) // srcIdx 0(旧) ss.WriteToSSTable([]LogEntry{tomb("k")}) // srcIdx 1(新,墓碑) diff --git a/storage/options.go b/storage/options.go new file mode 100644 index 0000000..baeb4d8 --- /dev/null +++ b/storage/options.go @@ -0,0 +1,64 @@ +package storage + +import "github.com/NeverENG/BanDB/config" + +// Options 是存储引擎的全部可调参数,构造时一次性传入。 +// +// 之所以显式传参而非在包内读全局 config.G:全局配置让同一进程内无法并存两套不同配置的 +// 引擎(多节点集成测试正需要),也让测试之间经由全局变量互相影响——本包此前正因如此 +// 出现过偶发失败。它还迫使生产代码写防御性代码:构造时把配置「快照」一份,以避开与测试 +// 中并发改配置形成的数据竞争。参数一旦由调用方传入,这些问题都不复存在。 +type Options struct { + // Dir 是 SSTable 文件目录。 + Dir string + + // MaxMemTableSize 是 active 表的条目数阈值,超过即触发 flush。 + MaxMemTableSize int + + // MaxCompactionSize 是单层文件数阈值,达到即触发该层 compaction。 + MaxCompactionSize int + + // MaxInflightBytes 是未 flush 数据(active + 正在 flush 的 dirty)的字节预算, + // 超出即阻塞写入等待 flush 归还信用。<=0 关闭背压。 + MaxInflightBytes int64 + + // BlockCacheBytes 是 SSTable 数据块缓存的字节预算。<=0 关闭缓存。 + BlockCacheBytes int64 + + // SkipListMaxLevel 与 SkipListP 是跳表的最大层高与升层概率。 + SkipListMaxLevel int + SkipListP float64 +} + +// DefaultOptions 从全局配置取一份参数。 +// +// 这是全局配置进入存储层的唯一入口:调用方在构造时读一次,此后引擎只认自己那份参数, +// 不再受 config.G 后续变动影响。 +func DefaultOptions() Options { + return Options{ + Dir: config.G.SSTablePath, + MaxMemTableSize: config.G.MaxMemTableSize, + MaxCompactionSize: config.G.MaxCompactionSize, + MaxInflightBytes: config.G.MemTableMaxInflightBytes, + BlockCacheBytes: config.G.BlockCacheBytes, + SkipListMaxLevel: config.G.MaxMemTableLevel, + SkipListP: config.G.MaxMemTableP, + } +} + +// withDefaults 补齐零值字段,使 Options{Dir: dir} 这样的部分构造也可用。 +func (o Options) withDefaults() Options { + if o.MaxMemTableSize <= 0 { + o.MaxMemTableSize = 1024 + } + if o.MaxCompactionSize <= 0 { + o.MaxCompactionSize = 4 + } + if o.SkipListMaxLevel <= 0 { + o.SkipListMaxLevel = 32 + } + if o.SkipListP <= 0 || o.SkipListP >= 1 { + o.SkipListP = 0.5 + } + return o +} diff --git a/storage/recency_fuzz_test.go b/storage/recency_fuzz_test.go index bd76853..69f885b 100644 --- a/storage/recency_fuzz_test.go +++ b/storage/recency_fuzz_test.go @@ -4,8 +4,6 @@ import ( "fmt" "testing" "time" - - "github.com/NeverENG/BanDB/config" ) // TestRecency_RandomizedOverwritesSurviveRestart 是 newest-wins 的随机化守卫: @@ -13,12 +11,9 @@ import ( // 然后模拟重启,逐一校验读到的都是最新值。这能捕捉 metas 重建顺序错误导致的陈旧值倒挂。 func TestRecency_RandomizedOverwritesSurviveRestart(t *testing.T) { dir := t.TempDir() - oldPath, oldComp := config.G.SSTablePath, config.G.MaxCompactionSize - config.G.SSTablePath = dir - config.G.MaxCompactionSize = 3 - defer func() { config.G.SSTablePath, config.G.MaxCompactionSize = oldPath, oldComp }() + opts := Options{Dir: dir, MaxCompactionSize: 3} - mt := newBareMemTable(NewSSTable()) + mt := newBareMemTable(NewSSTable(opts), opts) const keyspace = 40 ref := make(map[string]string) // 参考真值:key → 最新 value @@ -49,10 +44,10 @@ func TestRecency_RandomizedOverwritesSurviveRestart(t *testing.T) { } // 模拟重启。 - sst2 := NewSSTable() + sst2 := NewSSTable(opts) sst2.LoadSSTableMetaList() time.Sleep(200 * time.Millisecond) - mt2 := newBareMemTable(sst2) + mt2 := newBareMemTable(sst2, opts) stale := 0 for k, want := range ref { diff --git a/storage/recency_test.go b/storage/recency_test.go index a783f37..5f659c7 100644 --- a/storage/recency_test.go +++ b/storage/recency_test.go @@ -3,8 +3,6 @@ package storage import ( "testing" "time" - - "github.com/NeverENG/BanDB/config" ) // TestRecency_OverwriteAcrossCompactionSurvivesRestart 是「newest-wins 跨重启不倒挂」的 @@ -15,12 +13,9 @@ import ( // 排到新的 L0 文件之后,逆序就会先命中旧 merged,返回陈旧值 A。 func TestRecency_OverwriteAcrossCompactionSurvivesRestart(t *testing.T) { dir := t.TempDir() - oldPath, oldComp := config.G.SSTablePath, config.G.MaxCompactionSize - config.G.SSTablePath = dir - config.G.MaxCompactionSize = 2 - defer func() { config.G.SSTablePath, config.G.MaxCompactionSize = oldPath, oldComp }() + opts := Options{Dir: dir, MaxCompactionSize: 2} - mt := newBareMemTable(NewSSTable()) + mt := newBareMemTable(NewSSTable(opts), opts) // x=A 与另一个 key 一起落 L0,再补一个 L0,触发 compaction 把它们并成 L1 merged 文件。 if err := mt.FlushToSSTable([]LogEntry{ @@ -44,10 +39,10 @@ func TestRecency_OverwriteAcrossCompactionSurvivesRestart(t *testing.T) { } // 模拟重启:全新 SSTable 从磁盘恢复。 - sst2 := NewSSTable() + sst2 := NewSSTable(opts) sst2.LoadSSTableMetaList() time.Sleep(150 * time.Millisecond) - mt2 := newBareMemTable(sst2) + mt2 := newBareMemTable(sst2, opts) v, ok := mt2.getFromSSTables([]byte("x")) if !ok { diff --git a/storage/reload_recover_test.go b/storage/reload_recover_test.go index 67658fc..2a52d82 100644 --- a/storage/reload_recover_test.go +++ b/storage/reload_recover_test.go @@ -4,8 +4,6 @@ import ( "fmt" "testing" "time" - - "github.com/NeverENG/BanDB/config" ) // TestReloadRecoversFlushedKeys 守护 SSTable 重载恢复:写入远超单表阈值的数据逼其 flush 到 @@ -15,21 +13,11 @@ import ( // 退化成空串,使 [MinKey,MaxKey] 过滤把所有命中 key 跳过 → 重启后已 flush 数据全部丢失。 // 现在 MaxKey 只从块索引取,取不到则不施加上界。 func TestReloadRecoversFlushedKeys(t *testing.T) { - oldWAL := config.G.WALPath - oldSST := config.G.SSTablePath - oldMax := config.G.MaxMemTableSize dir := t.TempDir() - config.G.WALPath = dir + "/wal.log" - config.G.SSTablePath = dir - config.G.MaxMemTableSize = 4 // 极小阈值,逼迫多轮 active→dirty→SSTable flush - defer func() { - config.G.WALPath = oldWAL - config.G.SSTablePath = oldSST - config.G.MaxMemTableSize = oldMax - }() + opts := Options{Dir: dir, MaxMemTableSize: 4} // 极小阈值,逼迫多轮 active→dirty→SSTable flush const n = 50 - mt := NewEngine() + mt := NewEngine(opts) for i := 0; i < n; i++ { key := []byte(fmt.Sprintf("k%04d", i)) if err := mt.Put(key, []byte(fmt.Sprintf("v%04d", i))); err != nil { @@ -40,7 +28,7 @@ func TestReloadRecoversFlushedKeys(t *testing.T) { _ = mt.Close() // 停后台协程,避免与重载实例抢同一目录 // 模拟重启:同一目录新建 MemTable,从 SSTable 重新加载。 - mt2 := NewEngine() + mt2 := NewEngine(opts) defer mt2.Close() time.Sleep(200 * time.Millisecond) // 等 SSTable 元数据/索引异步加载 @@ -69,17 +57,11 @@ func TestReloadRecoversFlushedKeys(t *testing.T) { // // 本用例故意不 sleep:一旦加载退回异步,它就会失败。 func TestNewEngineSeesExistingSSTablesImmediately(t *testing.T) { - dir := t.TempDir() - oldWAL, oldSST, oldMax := config.G.WALPath, config.G.SSTablePath, config.G.MaxMemTableSize - config.G.WALPath = dir + "/wal.log" - config.G.SSTablePath = dir - config.G.MaxMemTableSize = 1 << 20 // 足够大:本用例自行写出 SSTable,不依赖自动 flush - t.Cleanup(func() { - config.G.WALPath, config.G.SSTablePath, config.G.MaxMemTableSize = oldWAL, oldSST, oldMax - }) + // 阈值取大:本用例自行写出 SSTable,不依赖自动 flush。 + opts := Options{Dir: t.TempDir(), MaxMemTableSize: 1 << 20} // 先在目录里放好一个 SSTable。 - seed := NewSSTable() + seed := NewSSTable(opts) entries := make([]LogEntry, 0, 64) for i := 0; i < 64; i++ { entries = append(entries, LogEntry{ @@ -91,7 +73,7 @@ func TestNewEngineSeesExistingSSTablesImmediately(t *testing.T) { t.Fatalf("WriteToSSTable: %v", err) } - e := NewEngine() + e := NewEngine(opts) t.Cleanup(func() { e.Close() }) // 立即读,不给后台加载留任何时间窗口。 diff --git a/storage/scan_test.go b/storage/scan_test.go index 4586079..7562025 100644 --- a/storage/scan_test.go +++ b/storage/scan_test.go @@ -20,7 +20,7 @@ func newMemWith(active, dirty *SkipList) *Engine { } func sl(pairs ...[2]string) *SkipList { - s := newSkipList() + s := newSkipList(32, 0.5) for _, p := range pairs { var val []byte if p[1] != "" { diff --git a/storage/skiplist.go b/storage/skiplist.go index 60c7e73..577b97e 100644 --- a/storage/skiplist.go +++ b/storage/skiplist.go @@ -14,7 +14,12 @@ type SkipList struct { size int level int head *SkipNode - byteSize int64 // 当前表内 key+value 的累计字节数(覆盖写按增量维护) + byteSize int64 + + // maxLevel 与 p 是本表的层高上限与升层概率,构造时固定。此前它们是包级变量且在 + // import 时取自全局配置,既无法按实例配置,也让测试改配置成为空操作。 + maxLevel int + p float64 // 当前表内 key+value 的累计字节数(覆盖写按增量维护) } // SkipNode 是跳表节点。Next 的长度即该节点的层高。 @@ -25,9 +30,11 @@ type SkipNode struct { } // newSkipList 创建一个新的空跳表 -func newSkipList() *SkipList { +func newSkipList(maxLevel int, p float64) *SkipList { return &SkipList{ - head: newSkipNode(maxLevel, nil, nil), + head: newSkipNode(maxLevel, nil, nil), + maxLevel: maxLevel, + p: p, } } @@ -40,10 +47,10 @@ func newSkipNode(level int, key []byte, value []byte) *SkipNode { } } -// randomLevel 生成随机层级 -func randomLevel() int { +// randomLevel 生成新节点的随机层级。 +func (sl *SkipList) randomLevel() int { level := 1 - for rand.Float64() < probability && level < maxLevel { + for rand.Float64() < sl.p && level < sl.maxLevel { level++ } return level @@ -85,7 +92,7 @@ func (sl *SkipList) search(key []byte) ([]byte, bool) { // Put 插入或更新键值对,始终操作 active 表 func (sl *SkipList) insert(key []byte, value []byte) int64 { - update := make([]*SkipNode, maxLevel) + update := make([]*SkipNode, sl.maxLevel) p := sl.head for i := sl.level - 1; i >= 0; i-- { @@ -106,7 +113,7 @@ func (sl *SkipList) insert(key []byte, value []byte) int64 { } // 生成新节点的随机层级 - newLevel := randomLevel() + newLevel := sl.randomLevel() if newLevel > sl.level { for i := sl.level; i < newLevel; i++ { update[i] = sl.head @@ -129,7 +136,7 @@ func (sl *SkipList) insert(key []byte, value []byte) int64 { // Delete 删除指定 key 的节点,始终操作 active 表 func (sl *SkipList) delete(key []byte) bool { - update := make([]*SkipNode, maxLevel) + update := make([]*SkipNode, sl.maxLevel) p := sl.head for i := sl.level - 1; i >= 0; i-- { diff --git a/storage/sstable.go b/storage/sstable.go index 95e40eb..2f094ff 100644 --- a/storage/sstable.go +++ b/storage/sstable.go @@ -3,7 +3,6 @@ package storage import ( - "github.com/NeverENG/BanDB/config" "os" "sync" "sync/atomic" @@ -80,14 +79,16 @@ type SSTable struct { blocks *blockCache } -func NewSSTable() *SSTable { +// NewSSTable 按 opts 构造 SSTable 集合的管理者。 +func NewSSTable(opts Options) *SSTable { + opts = opts.withDefaults() ss := &SSTable{ - dir: config.G.SSTablePath, + dir: opts.Dir, metas: make([]*SSTableMeta, 0), indexCache: make(map[string]*blockIndex), bloomCache: make(map[string]*PartitionedBloom), fdCache: make(map[string]*os.File), - blocks: newBlockCache(config.G.BlockCacheBytes), + blocks: newBlockCache(opts.BlockCacheBytes), } ss.publishMetas() return ss diff --git a/storage/sstable_bloom_test.go b/storage/sstable_bloom_test.go index 22113f6..d9a3824 100644 --- a/storage/sstable_bloom_test.go +++ b/storage/sstable_bloom_test.go @@ -8,8 +8,6 @@ import ( "os" "path/filepath" "testing" - - "github.com/NeverENG/BanDB/config" ) // makeSortedEntries 生成跨多个仓库、整体有序的 entries。 @@ -72,19 +70,17 @@ func writeV1SSTable(t *testing.T, path string, entries []LogEntry) { // TestSSTableBloomRoundTrip 写入(含布隆)→读取:存在的 key 命中, // 不存在的 key 返回 false,且布隆过滤器被正确写入并加载。 func TestSSTableBloomRoundTrip(t *testing.T) { - old := config.G.SSTablePath - config.G.SSTablePath = t.TempDir() - defer func() { config.G.SSTablePath = old }() + opts := testOptions(t) entries := makeSortedEntries(200) - ss := NewSSTable() + ss := NewSSTable(opts) if err := ss.WriteToSSTable(entries); err != nil { t.Fatalf("write: %v", err) } path := ss.Metas()[0].Filepath // 布隆过滤器已写入(从磁盘重新加载验证,绕过写入缓存) - fresh := NewSSTable() + fresh := NewSSTable(opts) if bloom := fresh.getBloom(path); bloom == nil { t.Fatal("expected bloom filter in v2 file, got nil") } @@ -107,16 +103,14 @@ func TestSSTableBloomRoundTrip(t *testing.T) { // TestSSTableBackwardCompatV1 旧格式文件(无布隆)仍能被正确读取, // 且 getBloom 返回 nil(不误判为有布隆)。 func TestSSTableBackwardCompatV1(t *testing.T) { - old := config.G.SSTablePath dir := t.TempDir() - config.G.SSTablePath = dir - defer func() { config.G.SSTablePath = old }() + opts := Options{Dir: dir} entries := makeSortedEntries(80) path := filepath.Join(dir, "sstable_v1.sst") writeV1SSTable(t, path, entries) - ss := NewSSTable() + ss := NewSSTable(opts) if bloom := ss.getBloom(path); bloom != nil { t.Fatal("v1 file must have no bloom, got non-nil") } diff --git a/storage/sstable_truncated_test.go b/storage/sstable_truncated_test.go index d599cb1..1c1fa2d 100644 --- a/storage/sstable_truncated_test.go +++ b/storage/sstable_truncated_test.go @@ -3,10 +3,7 @@ package storage import ( "fmt" "os" - "path/filepath" "testing" - - "github.com/NeverENG/BanDB/config" ) // TestTruncatedTailStillReadable 验证尾部残缺的 SSTable 仍能读到其数据段里的 key。 @@ -18,13 +15,10 @@ import ( // // 现在的约定是:MaxKey 不可信就不施加上界。多扫一个文件是可接受的代价,漏读不是。 func TestTruncatedTailStillReadable(t *testing.T) { - dir := t.TempDir() - oldSST := config.G.SSTablePath - config.G.SSTablePath = dir - t.Cleanup(func() { config.G.SSTablePath = oldSST }) + opts := testOptions(t) // 写一个正常的 SSTable(含完整尾部)。 - ss := NewSSTable() + ss := NewSSTable(opts) entries := make([]LogEntry, 0, 200) for i := 0; i < 200; i++ { entries = append(entries, LogEntry{ @@ -47,7 +41,7 @@ func TestTruncatedTailStillReadable(t *testing.T) { } // 模拟重启:重新加载元信息(此时读不到 footer)。 - fresh := NewSSTable() + fresh := NewSSTable(opts) fresh.LoadSSTableMetaList() metas := fresh.Metas() if len(metas) != 1 { @@ -76,16 +70,9 @@ func TestTruncatedTailStillReadable(t *testing.T) { // 范围过滤整段跳过。这是上一个用例的端到端版本——ReadFromSSTable 直接按路径读,绕过了 // 范围过滤,而真正出问题的正是过滤那一步。 func TestTruncatedTailNotSkippedByRangeFilter(t *testing.T) { - dir := t.TempDir() - oldSST, oldWAL, oldMax := config.G.SSTablePath, config.G.WALPath, config.G.MaxMemTableSize - config.G.SSTablePath = dir - config.G.WALPath = filepath.Join(dir, "wal.log") - config.G.MaxMemTableSize = 4 // 极小阈值,逼迫尽快 flush 出 SSTable - t.Cleanup(func() { - config.G.SSTablePath, config.G.WALPath, config.G.MaxMemTableSize = oldSST, oldWAL, oldMax - }) + opts := Options{Dir: t.TempDir(), MaxMemTableSize: 4} - ss := NewSSTable() + ss := NewSSTable(opts) entries := []LogEntry{ {Key: []byte("aaa"), Value: []byte("v-aaa")}, {Key: []byte("mmm"), Value: []byte("v-mmm")}, @@ -101,7 +88,7 @@ func TestTruncatedTailNotSkippedByRangeFilter(t *testing.T) { } // 引擎重启:其内部会 LoadSSTableMetaList,随后 Get 走 [MinKey,MaxKey] 过滤。 - e := NewEngine() + e := NewEngine(opts) t.Cleanup(func() { e.Close() }) // zzz 是排序最靠后的 key:上界一旦被猜小,它必然第一个被漏掉。 diff --git a/storage/testhelpers_test.go b/storage/testhelpers_test.go index d7213b5..17a5944 100644 --- a/storage/testhelpers_test.go +++ b/storage/testhelpers_test.go @@ -1,14 +1,25 @@ package storage -import "github.com/NeverENG/BanDB/config" +import "testing" -// newBareMemTable 构造一个不启动后台 goroutine 的 MemTable(供确定性测试直接驱动 -// FlushToSSTable/CompactSSTable),并从当前 config 快照 flush/compaction 阈值—— -// 与生产 NewEngine 一致,使 CompactSSTable 的阈值判断走 m.maxCompaction 而非全局。 -func newBareMemTable(sst *SSTable) *Engine { +// newBareMemTable 构造一个不启动后台 goroutine 的引擎(供确定性测试直接驱动 +// FlushToSSTable/CompactSSTable),参数取自传入的 opts,与生产 NewEngine 一致。 +func newBareMemTable(sst *SSTable, opts Options) *Engine { + opts = opts.withDefaults() return &Engine{ + active: newSkipList(opts.SkipListMaxLevel, opts.SkipListP), sst: sst, - maxSize: config.G.MaxMemTableSize, - maxCompaction: config.G.MaxCompactionSize, + maxSize: opts.MaxMemTableSize, + maxCompaction: opts.MaxCompactionSize, + opts: opts, } } + +// testOptions 返回一份指向独立临时目录的引擎参数。 +// +// 测试一律用它构造引擎,不再改 config.G:全局配置被并发运行的用例与它们各自的后台协程 +// 共享,此前正因如此出现过跨用例干扰与偶发失败。参数各自独立后,用例之间不再有隐式耦合。 +func testOptions(t *testing.T) Options { + t.Helper() + return Options{Dir: t.TempDir()} +}