From e3440e1b4a745a6191bfd52d939564dd9418fecb Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 11 Jul 2022 17:42:42 +1000 Subject: [PATCH 1/5] Merge CoinDb implementations --- .../Coindb/{LeveldbCoindb.cs => Coindb.cs} | 88 ++-- .../CoinViews/Coindb/RocksDbCoindb.cs | 385 ------------------ .../FullNodeBuilderConsensusExtension.cs | 7 +- .../NodeContext.cs | 5 +- src/Stratis.Bitcoin/Database/IDb.cs | 322 +++++++++++++++ src/Stratis.Bitcoin/Database/LevelDb.cs | 127 ++++++ src/Stratis.Bitcoin/Database/RocksDb.cs | 127 ++++++ 7 files changed, 627 insertions(+), 434 deletions(-) rename src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/{LeveldbCoindb.cs => Coindb.cs} (76%) delete mode 100644 src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/RocksDbCoindb.cs create mode 100644 src/Stratis.Bitcoin/Database/IDb.cs create mode 100644 src/Stratis.Bitcoin/Database/LevelDb.cs create mode 100644 src/Stratis.Bitcoin/Database/RocksDb.cs diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/LeveldbCoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs similarity index 76% rename from src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/LeveldbCoindb.cs rename to src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs index 2584bab096..cb536974d0 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/LeveldbCoindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs @@ -2,11 +2,11 @@ using System.Collections.Generic; using System.Linq; using System.Text; -using LevelDB; using Microsoft.Extensions.Logging; using NBitcoin; using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Configuration.Logging; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.Features.Consensus.CoinViews @@ -14,7 +14,7 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews /// /// Persistent implementation of coinview using the dBreeze database engine. /// - public class LevelDbCoindb : ICoindb, IStakedb, IDisposable + public class Coindb : ICoindb, IStakedb, IDisposable where T : IDb, new() { /// Database key under which the block hash of the coin view's current tip is stored. private static readonly byte[] blockHashKey = new byte[0]; @@ -41,19 +41,19 @@ public class LevelDbCoindb : ICoindb, IStakedb, IDisposable private BackendPerformanceSnapshot latestPerformanceSnapShot; /// Access to dBreeze database. - private DB leveldb; + private IDb leveldb; private readonly DBreezeSerializer dBreezeSerializer; private const int MaxRewindBatchSize = 10000; - public LevelDbCoindb(Network network, DataFolder dataFolder, IDateTimeProvider dateTimeProvider, + public Coindb(Network network, DataFolder dataFolder, IDateTimeProvider dateTimeProvider, INodeStats nodeStats, DBreezeSerializer dBreezeSerializer) : this(network, dataFolder.CoindbPath, dateTimeProvider, nodeStats, dBreezeSerializer) { } - public LevelDbCoindb(Network network, string dataFolder, IDateTimeProvider dateTimeProvider, + public Coindb(Network network, string dataFolder, IDateTimeProvider dateTimeProvider, INodeStats nodeStats, DBreezeSerializer dBreezeSerializer) { Guard.NotNull(network, nameof(network)); @@ -72,38 +72,38 @@ public LevelDbCoindb(Network network, string dataFolder, IDateTimeProvider dateT public void Initialize(ChainedHeader chainTip) { // Open a connection to a new DB and create if not found - var options = new Options { CreateIfMissing = true }; - this.leveldb = new DB(options, this.dataFolder); + this.leveldb = new T(); + this.leveldb.Open(this.dataFolder); // Check if key bytes are in the wrong endian order. HashHeightPair current = this.GetTipHash(); if (current != null) { - byte[] row = this.leveldb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(current.Height)).ToArray()); + byte[] row = this.leveldb.Get(rewindTable, BitConverter.GetBytes(current.Height)); // Fix the table if required. if (row != null) { // To be sure, check the next height too. - byte[] row2 = (current.Height > 1) ? this.leveldb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(current.Height - 1)).ToArray()) : new byte[] { }; + byte[] row2 = (current.Height > 1) ? this.leveldb.Get(rewindTable, BitConverter.GetBytes(current.Height - 1)) : new byte[] { }; if (row2 != null) { this.logger.LogInformation("Fixing the coin db."); var rows = new Dictionary(); - using (var iterator = this.leveldb.CreateIterator()) + using (var iterator = this.leveldb.GetIterator(rewindTable)) { - iterator.Seek(new byte[] { rewindTable }); + iterator.Seek(new byte[0]); while (iterator.IsValid()) { byte[] key = iterator.Key(); - if (key.Length != 5 || key[0] != rewindTable) + if (key.Length != 4) break; - int height = BitConverter.ToInt32(key, 1); + int height = BitConverter.ToInt32(key); rows[height] = iterator.Value(); @@ -111,19 +111,19 @@ public void Initialize(ChainedHeader chainTip) } } - using (var batch = new WriteBatch()) + using (var batch = this.leveldb.GetWriteBatch()) { foreach (int height in rows.Keys.OrderBy(k => k)) { - batch.Delete(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height)).ToArray()); + batch.Delete(rewindTable, BitConverter.GetBytes(height)); } foreach (int height in rows.Keys.OrderBy(k => k)) { - batch.Put(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray(), rows[height]); + batch.Put(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray(), rows[height]); } - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } } } @@ -135,10 +135,10 @@ public void Initialize(ChainedHeader chainTip) if (this.GetTipHash() == null) { - using (var batch = new WriteBatch()) + using (var batch = this.leveldb.GetWriteBatch()) { this.SetBlockHash(batch, new HashHeightPair(genesis.GetHash(), 0)); - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } } @@ -156,7 +156,7 @@ private void EnsureCoinDatabaseIntegrity(ChainedHeader chainTip) { heightToCheck += 1; - byte[] row = this.leveldb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(heightToCheck).Reverse()).ToArray()); + byte[] row = this.leveldb.Get(rewindTable, BitConverter.GetBytes(heightToCheck).Reverse().ToArray()); if (row == null) break; @@ -173,17 +173,17 @@ private void EnsureCoinDatabaseIntegrity(ChainedHeader chainTip) this.logger.LogInformation("Coin database integrity good."); } - private void SetBlockHash(WriteBatch batch, HashHeightPair nextBlockHash) + private void SetBlockHash(IDbBatch batch, HashHeightPair nextBlockHash) { this.persistedCoinviewTip = nextBlockHash; - batch.Put(new byte[] { blockTable }.Concat(blockHashKey).ToArray(), nextBlockHash.ToBytes()); + batch.Put(blockTable, blockHashKey, nextBlockHash.ToBytes()); } public HashHeightPair GetTipHash() { if (this.persistedCoinviewTip == null) { - var row = this.leveldb.Get(new byte[] { blockTable }.Concat(blockHashKey).ToArray()); + var row = this.leveldb.Get(blockTable, blockHashKey); if (row != null) { this.persistedCoinviewTip = new HashHeightPair(); @@ -204,7 +204,7 @@ public FetchCoinsResponse FetchCoins(OutPoint[] utxos) foreach (OutPoint outPoint in utxos) { - byte[] row = this.leveldb.Get(new byte[] { coinsTable }.Concat(outPoint.ToBytes()).ToArray()); + byte[] row = this.leveldb.Get(coinsTable, outPoint.ToBytes()); Coins outputs = row != null ? this.dBreezeSerializer.Deserialize(row) : null; this.logger.LogDebug("Outputs for '{0}' were {1}.", outPoint, outputs == null ? "NOT loaded" : "loaded"); @@ -220,7 +220,7 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB { int insertedEntities = 0; - using (var batch = new WriteBatch()) + using (var batch = this.leveldb.GetWriteBatch()) { using (new StopwatchDisposable(o => this.performanceCounter.AddInsertTime(o))) { @@ -239,7 +239,7 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB if (coin.Coins == null) { this.logger.LogDebug("Outputs of transaction ID '{0}' are prunable and will be removed from the database.", coin.OutPoint); - batch.Delete(new byte[] { coinsTable }.Concat(coin.OutPoint.ToBytes()).ToArray()); + batch.Delete(coinsTable, coin.OutPoint.ToBytes()); } else { @@ -254,7 +254,7 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB var coin = toInsert[i]; this.logger.LogDebug("Outputs of transaction ID '{0}' are NOT PRUNABLE and will be inserted into the database. {1}/{2}.", coin.OutPoint, i, toInsert.Count); - batch.Put(new byte[] { coinsTable }.Concat(coin.OutPoint.ToBytes()).ToArray(), this.dBreezeSerializer.Serialize(coin.Coins)); + batch.Put(coinsTable, coin.OutPoint.ToBytes(), this.dBreezeSerializer.Serialize(coin.Coins)); } if (rewindDataList != null) @@ -265,13 +265,13 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB this.logger.LogDebug("Rewind state #{0} created.", nextRewindIndex); - batch.Put(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(nextRewindIndex).Reverse()).ToArray(), this.dBreezeSerializer.Serialize(rewindData)); + batch.Put(rewindTable, BitConverter.GetBytes(nextRewindIndex).Reverse().ToArray(), this.dBreezeSerializer.Serialize(rewindData)); } } insertedEntities += unspentOutputs.Count; this.SetBlockHash(batch, nextBlockHash); - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } } @@ -282,18 +282,18 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB public int GetMinRewindHeight() { // Find the first row with a rewind table key prefix. - using (var iterator = this.leveldb.CreateIterator()) + using (var iterator = this.leveldb.GetIterator(rewindTable)) { - iterator.Seek(new byte[] { rewindTable }); + iterator.Seek(new byte[0]); if (!iterator.IsValid()) return -1; byte[] key = iterator.Key(); - if (key.Length != 5 || key[0] != rewindTable) + if (key.Length != 4) return -1; - return BitConverter.ToInt32(key.SafeSubarray(1, 4).Reverse().ToArray()); + return BitConverter.ToInt32(key.SafeSubarray(0, 4).Reverse().ToArray()); } } @@ -308,36 +308,36 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) { HashHeightPair res = null; - using (var batch = new WriteBatch()) + using (var batch = this.leveldb.GetWriteBatch()) { for (int height = startHeight; height > (target?.Height ?? (startHeight - 1)) && height > (startHeight - MaxRewindBatchSize); height--) { - byte[] row = this.leveldb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray()); + byte[] row = this.leveldb.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); if (row == null) throw new InvalidOperationException($"No rewind data found for block at height {height}."); - batch.Delete(BitConverter.GetBytes(height)); + batch.Delete(rewindTable, BitConverter.GetBytes(height)); var rewindData = this.dBreezeSerializer.Deserialize(row); foreach (OutPoint outPoint in rewindData.OutputsToRemove) { this.logger.LogDebug("Outputs of outpoint '{0}' will be removed.", outPoint); - batch.Delete(new byte[] { coinsTable }.Concat(outPoint.ToBytes()).ToArray()); + batch.Delete(coinsTable, outPoint.ToBytes()); } foreach (RewindDataOutput rewindDataOutput in rewindData.OutputsToRestore) { this.logger.LogDebug("Outputs of outpoint '{0}' will be restored.", rewindDataOutput.OutPoint); - batch.Put(new byte[] { coinsTable }.Concat(rewindDataOutput.OutPoint.ToBytes()).ToArray(), this.dBreezeSerializer.Serialize(rewindDataOutput.Coins)); + batch.Put(coinsTable, rewindDataOutput.OutPoint.ToBytes(), this.dBreezeSerializer.Serialize(rewindDataOutput.Coins)); } res = rewindData.PreviousBlockHash; } this.SetBlockHash(batch, res); - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } return res; @@ -345,7 +345,7 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) public RewindData GetRewindData(int height) { - byte[] row = this.leveldb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray()); + byte[] row = this.leveldb.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); return row != null ? this.dBreezeSerializer.Deserialize(row) : null; } @@ -355,18 +355,18 @@ public RewindData GetRewindData(int height) /// List of POS block information to be examined and persists if unsaved. public void PutStake(IEnumerable stakeEntries) { - using (var batch = new WriteBatch()) + using (var batch = this.leveldb.GetWriteBatch()) { foreach (StakeItem stakeEntry in stakeEntries) { if (!stakeEntry.InStore) { - batch.Put(new byte[] { stakeTable }.Concat(stakeEntry.BlockId.ToBytes(false)).ToArray(), this.dBreezeSerializer.Serialize(stakeEntry.BlockStake)); + batch.Put(stakeTable, stakeEntry.BlockId.ToBytes(false), this.dBreezeSerializer.Serialize(stakeEntry.BlockStake)); stakeEntry.InStore = true; } } - this.leveldb.Write(batch, new WriteOptions() { Sync = true }); + batch.Write(); } } @@ -379,7 +379,7 @@ public void GetStake(IEnumerable blocklist) foreach (StakeItem blockStake in blocklist) { this.logger.LogTrace("Loading POS block hash '{0}' from the database.", blockStake.BlockId); - byte[] stakeRow = this.leveldb.Get(new byte[] { stakeTable }.Concat(blockStake.BlockId.ToBytes(false)).ToArray()); + byte[] stakeRow = this.leveldb.Get(stakeTable, blockStake.BlockId.ToBytes(false)); if (stakeRow != null) { diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/RocksDbCoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/RocksDbCoindb.cs deleted file mode 100644 index 6e55bcd059..0000000000 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/RocksDbCoindb.cs +++ /dev/null @@ -1,385 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Microsoft.Extensions.Logging; -using NBitcoin; -using RocksDbSharp; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Configuration.Logging; -using Stratis.Bitcoin.Utilities; - -namespace Stratis.Bitcoin.Features.Consensus.CoinViews -{ - /// - /// Persistent implementation of coinview using dBreeze database. - /// - public class RocksDbCoindb : ICoindb, IStakedb, IDisposable - { - /// Database key under which the block hash of the coin view's current tip is stored. - private static readonly byte[] blockHashKey = new byte[0]; - - private static readonly byte coinsTable = 1; - private static readonly byte blockTable = 2; - private static readonly byte rewindTable = 3; - private static readonly byte stakeTable = 4; - - private readonly string dataFolder; - - /// Hash of the block which is currently the tip of the coinview. - private HashHeightPair persistedCoinviewTip; - private readonly DBreezeSerializer dBreezeSerializer; - private DbOptions dbOptions; - private RocksDb rocksDb; - private BackendPerformanceSnapshot latestPerformanceSnapShot; - private readonly ILogger logger; - private readonly Network network; - private readonly BackendPerformanceCounter performanceCounter; - - public RocksDbCoindb( - Network network, - DataFolder dataFolder, - IDateTimeProvider dateTimeProvider, - INodeStats nodeStats, - DBreezeSerializer dBreezeSerializer) - { - this.dataFolder = dataFolder.CoindbPath; - this.dBreezeSerializer = dBreezeSerializer; - this.logger = LogManager.GetCurrentClassLogger(); - this.network = network; - this.performanceCounter = new BackendPerformanceCounter(dateTimeProvider); - - if (nodeStats.DisplayBenchStats) - nodeStats.RegisterStats(this.AddBenchStats, StatsType.Benchmark, this.GetType().Name, 400); - } - - public void Initialize(ChainedHeader chainTip) - { - this.dbOptions = new DbOptions().SetCreateIfMissing(true); - this.rocksDb = RocksDb.Open(this.dbOptions, this.dataFolder); - - // Check if key bytes are in the wrong endian order. - HashHeightPair current = this.GetTipHash(); - if (current != null) - { - byte[] row = this.rocksDb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(current.Height)).ToArray()); - - // Fix the table if required. - if (row != null) - { - // To be sure, check the next height too. - byte[] row2 = (current.Height > 1) ? this.rocksDb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(current.Height - 1)).ToArray()) : new byte[] { }; - if (row2 != null) - { - this.logger.LogInformation("Fixing the coin db."); - - var rows = new Dictionary(); - - using (var iterator = this.rocksDb.NewIterator()) - { - iterator.Seek(new byte[] { rewindTable }); - - while (iterator.Valid()) - { - byte[] key = iterator.Key(); - - if (key.Length != 5 || key[0] != rewindTable) - break; - - int height = BitConverter.ToInt32(key, 1); - - rows[height] = iterator.Value(); - - iterator.Next(); - } - } - - using (var batch = new WriteBatch()) - { - foreach (int height in rows.Keys.OrderBy(k => k)) - { - batch.Delete(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height)).ToArray()); - } - - foreach (int height in rows.Keys.OrderBy(k => k)) - { - batch.Put(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray(), rows[height]); - } - - this.rocksDb.Write(batch); - } - } - } - } - - EnsureCoinDatabaseIntegrity(chainTip); - - Block genesis = this.network.GetGenesis(); - - if (this.GetTipHash() == null) - this.SetBlockHash(new HashHeightPair(genesis.GetHash(), 0)); - - this.logger.LogInformation("Coinview initialized with tip '{0}'.", this.persistedCoinviewTip); - } - - private void EnsureCoinDatabaseIntegrity(ChainedHeader chainTip) - { - this.logger.LogInformation("Checking coin database integrity..."); - - var heightToCheck = chainTip.Height; - - // Find the height up to where rewind data is stored above chain tip. - do - { - heightToCheck += 1; - - byte[] row = this.rocksDb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(heightToCheck).Reverse()).ToArray()); - if (row == null) - break; - - } while (true); - - using (var batch = new WriteBatch()) - { - for (int height = heightToCheck - 1; height > chainTip.Height; height--) - { - this.logger.LogInformation($"Fixing coin database, deleting rewind data at height {height} above tip '{chainTip}'."); - RewindInternal(batch, height); - } - } - - this.logger.LogInformation("Coin database integrity good."); - } - - private void SetBlockHash(HashHeightPair nextBlockHash) - { - this.persistedCoinviewTip = nextBlockHash; - this.rocksDb.Put(new byte[] { blockTable }.Concat(blockHashKey).ToArray(), nextBlockHash.ToBytes()); - } - - public HashHeightPair GetTipHash() - { - if (this.persistedCoinviewTip == null) - { - var row = this.rocksDb.Get(new byte[] { blockTable }.Concat(blockHashKey).ToArray()); - if (row != null) - { - this.persistedCoinviewTip = new HashHeightPair(); - this.persistedCoinviewTip.FromBytes(row); - } - } - - return this.persistedCoinviewTip; - } - - public FetchCoinsResponse FetchCoins(OutPoint[] utxos) - { - FetchCoinsResponse res = new FetchCoinsResponse(); - - using (new StopwatchDisposable(o => this.performanceCounter.AddQueryTime(o))) - { - this.performanceCounter.AddQueriedEntities(utxos.Length); - - foreach (OutPoint outPoint in utxos) - { - byte[] row = this.rocksDb.Get(new byte[] { coinsTable }.Concat(outPoint.ToBytes()).ToArray()); - Coins outputs = row != null ? this.dBreezeSerializer.Deserialize(row) : null; - - this.logger.LogDebug("Outputs for '{0}' were {1}.", outPoint, outputs == null ? "NOT loaded" : "loaded"); - - res.UnspentOutputs.Add(outPoint, new UnspentOutput(outPoint, outputs)); - } - } - - return res; - } - - public void SaveChanges(IList unspentOutputs, HashHeightPair oldBlockHash, HashHeightPair nextBlockHash, List rewindDataList = null) - { - int insertedEntities = 0; - - using (var batch = new WriteBatch()) - { - using (new StopwatchDisposable(o => this.performanceCounter.AddInsertTime(o))) - { - HashHeightPair current = this.GetTipHash(); - if (current != oldBlockHash) - { - this.logger.LogError("(-)[BLOCKHASH_MISMATCH]"); - throw new InvalidOperationException("Invalid oldBlockHash"); - } - - // Here we'll add items to be inserted in a second pass. - List toInsert = new List(); - - foreach (var coin in unspentOutputs.OrderBy(utxo => utxo.OutPoint, new OutPointComparer())) - { - if (coin.Coins == null) - { - this.logger.LogDebug("Outputs of transaction ID '{0}' are prunable and will be removed from the database.", coin.OutPoint); - batch.Delete(new byte[] { coinsTable }.Concat(coin.OutPoint.ToBytes()).ToArray()); - } - else - { - // Add the item to another list that will be used in the second pass. - // This is for performance reasons: dBreeze is optimized to run the same kind of operations, sorted. - toInsert.Add(coin); - } - } - - for (int i = 0; i < toInsert.Count; i++) - { - var coin = toInsert[i]; - this.logger.LogDebug("Outputs of transaction ID '{0}' are NOT PRUNABLE and will be inserted into the database. {1}/{2}.", coin.OutPoint, i, toInsert.Count); - - batch.Put(new byte[] { coinsTable }.Concat(coin.OutPoint.ToBytes()).ToArray(), this.dBreezeSerializer.Serialize(coin.Coins)); - } - - if (rewindDataList != null) - { - foreach (RewindData rewindData in rewindDataList) - { - var nextRewindIndex = rewindData.PreviousBlockHash.Height + 1; - - this.logger.LogDebug("Rewind state #{0} created.", nextRewindIndex); - - batch.Put(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(nextRewindIndex).Reverse()).ToArray(), this.dBreezeSerializer.Serialize(rewindData)); - } - } - - insertedEntities += unspentOutputs.Count; - this.rocksDb.Write(batch); - - this.SetBlockHash(nextBlockHash); - } - } - - this.performanceCounter.AddInsertedEntities(insertedEntities); - } - - - /// - public int GetMinRewindHeight() - { - // Find the first row with a rewind table key prefix. - using (var iterator = this.rocksDb.NewIterator()) - { - iterator.Seek(new byte[] { rewindTable }); - if (!iterator.Valid()) - return -1; - - byte[] key = iterator.Key(); - - if (key.Length != 5 || key[0] != rewindTable) - return -1; - - return BitConverter.ToInt32(key.SafeSubarray(1, 4).Reverse().ToArray()); - } - } - - /// - public HashHeightPair Rewind(HashHeightPair target) - { - using (var batch = new WriteBatch()) - { - HashHeightPair current = this.GetTipHash(); - return RewindInternal(batch, current.Height); - } - } - - private HashHeightPair RewindInternal(WriteBatch batch, int height) - { - byte[] row = this.rocksDb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray()); - - if (row == null) - throw new InvalidOperationException($"No rewind data found for block at height {height}."); - - batch.Delete(BitConverter.GetBytes(height)); - - var rewindData = this.dBreezeSerializer.Deserialize(row); - - foreach (OutPoint outPoint in rewindData.OutputsToRemove) - { - this.logger.LogDebug("Outputs of outpoint '{0}' will be removed.", outPoint); - batch.Delete(new byte[] { coinsTable }.Concat(outPoint.ToBytes()).ToArray()); - } - - foreach (RewindDataOutput rewindDataOutput in rewindData.OutputsToRestore) - { - this.logger.LogDebug("Outputs of outpoint '{0}' will be restored.", rewindDataOutput.OutPoint); - batch.Put(new byte[] { coinsTable }.Concat(rewindDataOutput.OutPoint.ToBytes()).ToArray(), this.dBreezeSerializer.Serialize(rewindDataOutput.Coins)); - } - - this.rocksDb.Write(batch); - - this.SetBlockHash(rewindData.PreviousBlockHash); - - return rewindData.PreviousBlockHash; - } - - public RewindData GetRewindData(int height) - { - byte[] row = this.rocksDb.Get(new byte[] { rewindTable }.Concat(BitConverter.GetBytes(height).Reverse()).ToArray()); - return row != null ? this.dBreezeSerializer.Deserialize(row) : null; - } - - /// - /// Persists unsaved POS blocks information to the database. - /// - /// List of POS block information to be examined and persists if unsaved. - public void PutStake(IEnumerable stakeEntries) - { - using var batch = new WriteBatch(); - { - foreach (StakeItem stakeEntry in stakeEntries) - { - if (!stakeEntry.InStore) - { - batch.Put(new byte[] { stakeTable }.Concat(stakeEntry.BlockId.ToBytes(false)).ToArray(), this.dBreezeSerializer.Serialize(stakeEntry.BlockStake)); - stakeEntry.InStore = true; - } - } - - this.rocksDb.Write(batch); - } - } - - /// - /// Retrieves POS blocks information from the database. - /// - /// List of partially initialized POS block information that is to be fully initialized with the values from the database. - public void GetStake(IEnumerable blocklist) - { - foreach (StakeItem blockStake in blocklist) - { - this.logger.LogDebug("Loading POS block hash '{0}' from the database.", blockStake.BlockId); - byte[] stakeRow = this.rocksDb.Get(new byte[] { stakeTable }.Concat(blockStake.BlockId.ToBytes(false)).ToArray()); - - if (stakeRow != null) - { - blockStake.BlockStake = this.dBreezeSerializer.Deserialize(stakeRow); - blockStake.InStore = true; - } - } - } - - private void AddBenchStats(StringBuilder log) - { - log.AppendLine("======RocksDb Bench======"); - - BackendPerformanceSnapshot snapShot = this.performanceCounter.Snapshot(); - - if (this.latestPerformanceSnapShot == null) - log.AppendLine(snapShot.ToString()); - else - log.AppendLine((snapShot - this.latestPerformanceSnapShot).ToString()); - - this.latestPerformanceSnapShot = snapShot; - } - - public void Dispose() - { - this.rocksDb.Dispose(); - } - } -} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs b/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs index fb3f89f155..06a751fb35 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/FullNodeBuilderConsensusExtension.cs @@ -4,6 +4,7 @@ using Stratis.Bitcoin.Builder; using Stratis.Bitcoin.Configuration.Logging; using Stratis.Bitcoin.Consensus; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.Consensus.CoinViews; using Stratis.Bitcoin.Features.Consensus.Interfaces; using Stratis.Bitcoin.Features.Consensus.ProvenBlockHeaders; @@ -87,15 +88,15 @@ public static void ConfigureCoinDatabaseImplementation(this IServiceCollection s break; case DbType.Leveldb: - services.AddSingleton(); + services.AddSingleton>(); break; case DbType.RocksDb: - services.AddSingleton(); + services.AddSingleton>(); break; default: - services.AddSingleton(); + services.AddSingleton>(); break; } } diff --git a/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs b/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs index 5efc05f99a..fb005f0b45 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/NodeContext.cs @@ -5,6 +5,7 @@ using Moq; using NBitcoin; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Database; using Stratis.Bitcoin.Features.Consensus.CoinViews; using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.Tests.Common; @@ -27,7 +28,7 @@ public NodeContext(object caller, string name, Network network) this.FolderName = TestBase.CreateTestDir(caller, name); var dateTimeProvider = new DateTimeProvider(); var serializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - this.Coindb = new LevelDbCoindb(network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(network), new Mock().Object), serializer); + this.Coindb = new Coindb(network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(network), new Mock().Object), serializer); this.Coindb.Initialize(new ChainedHeader(network.GetGenesis().Header, network.GenesisHash, 0)); this.cleanList = new List { (IDisposable)this.Coindb }; } @@ -65,7 +66,7 @@ public void ReloadPersistentCoinView(ChainedHeader chainTip) this.cleanList.Remove((IDisposable)this.Coindb); var dateTimeProvider = new DateTimeProvider(); var serializer = new DBreezeSerializer(this.Network.Consensus.ConsensusFactory); - this.Coindb = new LevelDbCoindb(this.Network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(this.Network), new Mock().Object), serializer); + this.Coindb = new Coindb(this.Network, this.FolderName, dateTimeProvider, new NodeStats(dateTimeProvider, NodeSettings.Default(this.Network), new Mock().Object), serializer); this.Coindb.Initialize(chainTip); this.cleanList.Add((IDisposable)this.Coindb); diff --git a/src/Stratis.Bitcoin/Database/IDb.cs b/src/Stratis.Bitcoin/Database/IDb.cs new file mode 100644 index 0000000000..5e16dd78e3 --- /dev/null +++ b/src/Stratis.Bitcoin/Database/IDb.cs @@ -0,0 +1,322 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NBitcoin; + +namespace Stratis.Bitcoin.Database +{ + /// + /// This interface and its relevant implementations provide a standardized interface to databases such as and , or other databases + /// capable of supporting key-based value retrieval and key iteration. + /// + /// + /// The interface expects keys to be specified as separate table and key identifiers. Similarly iterators are expected to be constrained to operate within single tables. + /// + public interface IDb : IDisposable + { + /// + /// Opens the database at the specified path. + /// + /// The path where the database is located. + void Open(string dbPath); + + /// + /// Gets the value associated with a table and key. + /// + /// The table identifier. + /// The key of the value to retrieve. + /// The value for the specified table and key. + byte[] Get(byte table, byte[] key); + + /// + /// Gets an iterator that allows iteration over keys in a table. + /// + /// The table that will be iterated. + /// See . + IDbIterator GetIterator(byte table); + + /// + /// Gets a batch that can be used to record changes that can be applied atomically. + /// + /// The method will not reflect these changes until they are committed. Use + /// the class if uncommitted changes need to be accessed. + /// See . + IDbBatch GetWriteBatch(); + + /// + /// Removes all tables and their contents. + /// + void Clear(); + } + + /// + /// A batch that can be used to record changes that can be applied atomically. + /// + /// The database's method will not reflect these changes until they are committed. + public interface IDbBatch : IDisposable + { + /// + /// Records a value that will be written to the database when the method is invoked. + /// + /// The table that will be updated. + /// The table key that identifies the value to be updated. + /// The value to be written to the table. + /// This class for fluent operations. + IDbBatch Put(byte table, byte[] key, byte[] value); + + /// + /// Records a key that will be deleted from the database when the method is invoked. + /// + /// The table that will be updated. + /// The table key that will be removed. + /// This class for fluent operations. + IDbBatch Delete(byte table, byte[] key); + + /// + /// Writes the recorded changes to the database. + /// + void Write(); + } + + /// + /// A batch that can be used to record changes that can be applied atomically. + /// + /// The supplied method will immediately reflect any changes that have + /// been made or retrieve the value from the underlying database. In contrast the database method + /// will only show the changes after the method is called. + public class ReadWriteBatch : IDbBatch + { + private readonly IDb db; + private readonly IDbBatch batch; + private Dictionary cache; + + public ReadWriteBatch(IDb db) + { + this.db = db; + this.batch = db.GetWriteBatch(); + this.cache = new Dictionary(new ByteArrayComparer()); + } + + /// + /// Records a value that will be written to the database when the method is invoked. + /// + /// The table that will be updated. + /// The table key that identifies the value to be updated. + /// The value to be written to the table. + /// This class for fluent operations. + public IDbBatch Put(byte table, byte[] key, byte[] value) + { + this.cache[new byte[] { table }.Concat(key).ToArray()] = value; + return this.batch.Put(table, key, value); + } + + /// + /// Records a key that will be deleted from the database when the method is invoked. + /// + /// The table that will be updated. + /// The table key that will be removed. + /// This interface for fluent operations. + public IDbBatch Delete(byte table, byte[] key) + { + this.cache[new byte[] { table }.Concat(key).ToArray()] = null; + return this.batch.Delete(table, key); + } + + /// + /// Returns any changes that have been made to the batch or retrieves the value from the underlying database.. + /// + /// The table of the value to be retrieved. + /// The table key of the value to retrieve. + /// This interface for fluent operations. + public byte[] Get(byte table, byte[] key) + { + if (this.cache.TryGetValue(new byte[] { table }.Concat(key).ToArray(), out byte[] value)) + return value; + + return this.db.Get(table, key); + } + + /// + /// Writes the recorded changes to the database. + /// + public void Write() + { + this.batch.Write(); + } + + public void Dispose() + { + this.batch.Dispose(); + } + } + + /// + /// Extension methods that build on the interface. + /// + public static class IDbExt + { + /// + /// Gets a . + /// + /// The database to get the batch for. + /// The . + public static ReadWriteBatch GetReadWriteBatch(this IDb db) + { + return new ReadWriteBatch(db); + } + } + + /// + /// An iterator that can be used to iterate the keys and values in an compliant database. + /// + public interface IDbIterator : IDisposable + { + /// + /// Seeks to a first key >= in the relevant table. + /// If no such key is found then will return false. + /// + /// The key to find. + void Seek(byte[] key); + + /// + /// Seeks to the last key in the relevant table. + /// If no such key is found then will return false. + /// + void SeekToLast(); + + /// + /// Seeks to the next key in the relevant table. + /// If no such key is found then will return false. + /// + void Next(); + + /// + /// Seeks to the previous key in the relevant table. + /// If no such key is found then will return false. + /// + void Prev(); + + /// + /// Determines if the current key is valid. + /// + /// true if a , , or operation found a valid key. false otherwise. + bool IsValid(); + + /// + /// The current key. + /// + /// The key. + byte[] Key(); + + /// + /// The current value. + /// + /// The value. + byte[] Value(); + } + + /// + /// Extension methods that build on the interface. + /// + public static class IDbIteratorExt + { + private static ByteArrayComparer byteArrayComparer = new ByteArrayComparer(); + + /// + /// Gets all the keys in the relevant table subject to any supplied constraints. + /// + /// The iterator that also identifies the table being iterated. + /// Defaults to false. Set to true if values should be ommitted - i.e. set to null. + /// Defaults to true. Set to false to return keys in ascending order. + /// Can be set optionally to specify the lower bound of keys to return. + /// Can be set optionally to specify the upper bound of keys to return. + /// Defaults to true. Set to false to omit the key specified in . + /// Defaults to true. Set to false to omit the key specified in . + /// An enumeration containing all the keys and values according to the specified constraints. + public static IEnumerable<(byte[], byte[])> GetAll(this IDbIterator iterator, bool keysOnly = false, bool ascending = true, + byte[] firstKey = null, byte[] lastKey = null, bool includeFirstKey = true, bool includeLastKey = true) + { + bool done = false; + Func breakLoop; + Action next; + + if (!ascending) + { + // Seek to the last key if it was provided. + if (lastKey == null) + iterator.SeekToLast(); + else + { + iterator.Seek(lastKey); + if (!(includeLastKey && iterator.IsValid() && byteArrayComparer.Equals(iterator.Key(), lastKey))) + iterator.Prev(); + } + + breakLoop = (firstKey == null) ? (Func)null : (keyBytes) => + { + int compareResult = byteArrayComparer.Compare(keyBytes, firstKey); + if (compareResult <= 0) + { + // If this is the first key and its not included or we've overshot the range then stop without yielding a value. + if (!includeFirstKey || compareResult < 0) + return true; + + // Stop after yielding the value. + done = true; + } + + // Keep going. + return false; + }; + + next = () => iterator.Prev(); + } + else /* Ascending */ + { + // Seek to the first key if it was provided. + if (firstKey == null) + iterator.Seek(new byte[0]); + else + { + iterator.Seek(firstKey); + if (!(includeFirstKey && iterator.IsValid() && byteArrayComparer.Equals(iterator.Key(), firstKey))) + iterator.Next(); + } + + breakLoop = (lastKey == null) ? (Func)null : (keyBytes) => + { + int compareResult = byteArrayComparer.Compare(keyBytes, lastKey); + if (compareResult >= 0) + { + // If this is the last key and its not included or we've overshot the range then stop without yielding a value. + if (!includeLastKey || compareResult > 0) + return true; + + // Stop after yielding the value. + done = true; + } + + // Keep going. + return false; + }; + + next = () => iterator.Next(); + } + + while (iterator.IsValid()) + { + byte[] keyBytes = iterator.Key(); + + if (breakLoop != null && breakLoop(keyBytes)) + break; + + yield return (keyBytes, keysOnly ? null : iterator.Value()); + + if (done) + break; + + next(); + } + } + } +} diff --git a/src/Stratis.Bitcoin/Database/LevelDb.cs b/src/Stratis.Bitcoin/Database/LevelDb.cs new file mode 100644 index 0000000000..a81e78486a --- /dev/null +++ b/src/Stratis.Bitcoin/Database/LevelDb.cs @@ -0,0 +1,127 @@ +using System.Linq; +using LevelDB; +using NBitcoin; + +namespace Stratis.Bitcoin.Database +{ + /// A minimal LevelDb wrapper that makes it compliant with the interface. + public class LevelDb : IDb + { + private string dbPath; + + private DB db; + + public IDbIterator GetIterator(byte table) + { + return new LevelDbIterator(table, this.db.CreateIterator()); + } + + public void Open(string dbPath) + { + this.dbPath = dbPath; + this.db = new DB(new Options() { CreateIfMissing = true }, dbPath); + } + + public void Clear() + { + this.db.Dispose(); + System.IO.Directory.Delete(this.dbPath, true); + this.db = new DB(new Options() { CreateIfMissing = true }, this.dbPath); + } + + public IDbBatch GetWriteBatch() => new LevelDbBatch(this.db); + + public byte[] Get(byte table, byte[] key) + { + return this.db.Get(new[] { table }.Concat(key).ToArray()); + } + + public void Dispose() + { + this.db.Dispose(); + } + } + + /// A minimal LevelDb wrapper that makes it compliant with the interface. + public class LevelDbBatch : WriteBatch, IDbBatch + { + private DB db; + + public LevelDbBatch(DB db) + { + this.db = db; + } + + public IDbBatch Put(byte table, byte[] key, byte[] value) + { + return (IDbBatch)this.Put(new[] { table }.Concat(key).ToArray(), value); + } + + public IDbBatch Delete(byte table, byte[] key) + { + return (IDbBatch)this.Delete(new[] { table }.Concat(key).ToArray()); + } + + public void Write() + { + this.db.Write(this, new WriteOptions() { Sync = true }); + } + } + + /// A minimal LevelDb wrapper that makes it compliant with the interface. + public class LevelDbIterator : IDbIterator + { + private byte table; + private Iterator iterator; + + public LevelDbIterator(byte table, Iterator iterator) + { + this.table = table; + this.iterator = iterator; + } + + public void Seek(byte[] key) + { + this.iterator.Seek(new[] { this.table }.Concat(key).ToArray()); + } + + public void SeekToLast() + { + this.iterator.Seek(new[] { (byte)(this.table + 1) }); + if (!this.iterator.IsValid()) + this.iterator.SeekToLast(); + else + this.iterator.Prev(); + } + + public void Next() + { + this.iterator.Next(); + } + + public void Prev() + { + this.iterator.Prev(); + } + + public bool IsValid() + { + return this.iterator.IsValid() && this.iterator.Key()[0] == this.table; + } + + public byte[] Key() + { + return this.iterator.Key().Skip(1).ToArray(); + } + + public byte[] Value() + { + return this.iterator.Value(); + } + + public void Dispose() + { + this.iterator.Dispose(); + } + } +} diff --git a/src/Stratis.Bitcoin/Database/RocksDb.cs b/src/Stratis.Bitcoin/Database/RocksDb.cs new file mode 100644 index 0000000000..f392c7e3b4 --- /dev/null +++ b/src/Stratis.Bitcoin/Database/RocksDb.cs @@ -0,0 +1,127 @@ +using System.Linq; +using NBitcoin; +using RocksDbSharp; + +namespace Stratis.Bitcoin.Database +{ + /// A minimal RocksDb wrapper that makes it compliant with the interface. + public class RocksDb : IDb + { + private string dbPath; + + private RocksDbSharp.RocksDb db; + + public IDbIterator GetIterator(byte table) + { + return new RocksDbIterator(table, this.db.NewIterator()); + } + + public void Open(string dbPath) + { + this.dbPath = dbPath; + this.db = RocksDbSharp.RocksDb.Open(new DbOptions().SetCreateIfMissing(), dbPath); + } + + public void Clear() + { + this.db.Dispose(); + System.IO.Directory.Delete(this.dbPath, true); + this.db = RocksDbSharp.RocksDb.Open(new DbOptions().SetCreateIfMissing(), this.dbPath); + } + + public IDbBatch GetWriteBatch() => new RocksDbBatch(this.db); + + public byte[] Get(byte table, byte[] key) + { + return this.db.Get(new[] { table }.Concat(key).ToArray()); + } + + public void Dispose() + { + this.db.Dispose(); + } + } + + /// A minimal RocksDb wrapper that makes it compliant with the interface. + public class RocksDbBatch : WriteBatch, IDbBatch + { + private RocksDbSharp.RocksDb db; + + public RocksDbBatch(RocksDbSharp.RocksDb db) + { + this.db = db; + } + + public IDbBatch Put(byte table, byte[] key, byte[] value) + { + return (IDbBatch)this.Put(new[] { table }.Concat(key).ToArray(), value); + } + + public IDbBatch Delete(byte table, byte[] key) + { + return (IDbBatch)this.Delete(new[] { table }.Concat(key).ToArray()); + } + + public void Write() + { + this.db.Write(this); + } + } + + /// A minimal RocksDb wrapper that makes it compliant with the interface. + public class RocksDbIterator : IDbIterator + { + private byte table; + private Iterator iterator; + + public RocksDbIterator(byte table, Iterator iterator) + { + this.table = table; + this.iterator = iterator; + } + + public void Seek(byte[] key) + { + this.iterator.Seek(new[] { this.table }.Concat(key).ToArray()); + } + + public void SeekToLast() + { + this.iterator.Seek(new[] { (byte)(this.table + 1) }); + if (!this.iterator.Valid()) + this.iterator.SeekToLast(); + else + this.iterator.Prev(); + } + + public void Next() + { + this.iterator.Next(); + } + + public void Prev() + { + this.iterator.Prev(); + } + + public bool IsValid() + { + return this.iterator.Valid() && this.iterator.Value()[0] == this.table; + } + + public byte[] Key() + { + return this.iterator.Key().Skip(1).ToArray(); + } + + public byte[] Value() + { + return this.iterator.Value(); + } + + public void Dispose() + { + this.iterator.Dispose(); + } + } +} From 639ac04d35c9b7fbd23d91cf124b3fbdbbae069e Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 11 Jul 2022 21:43:01 +1000 Subject: [PATCH 2/5] Reduce changes --- src/Stratis.Bitcoin/Database/IDb.cs | 198 +--------------------------- 1 file changed, 1 insertion(+), 197 deletions(-) diff --git a/src/Stratis.Bitcoin/Database/IDb.cs b/src/Stratis.Bitcoin/Database/IDb.cs index 5e16dd78e3..807271a29e 100644 --- a/src/Stratis.Bitcoin/Database/IDb.cs +++ b/src/Stratis.Bitcoin/Database/IDb.cs @@ -1,7 +1,4 @@ using System; -using System.Collections.Generic; -using System.Linq; -using NBitcoin; namespace Stratis.Bitcoin.Database { @@ -78,94 +75,6 @@ public interface IDbBatch : IDisposable void Write(); } - /// - /// A batch that can be used to record changes that can be applied atomically. - /// - /// The supplied method will immediately reflect any changes that have - /// been made or retrieve the value from the underlying database. In contrast the database method - /// will only show the changes after the method is called. - public class ReadWriteBatch : IDbBatch - { - private readonly IDb db; - private readonly IDbBatch batch; - private Dictionary cache; - - public ReadWriteBatch(IDb db) - { - this.db = db; - this.batch = db.GetWriteBatch(); - this.cache = new Dictionary(new ByteArrayComparer()); - } - - /// - /// Records a value that will be written to the database when the method is invoked. - /// - /// The table that will be updated. - /// The table key that identifies the value to be updated. - /// The value to be written to the table. - /// This class for fluent operations. - public IDbBatch Put(byte table, byte[] key, byte[] value) - { - this.cache[new byte[] { table }.Concat(key).ToArray()] = value; - return this.batch.Put(table, key, value); - } - - /// - /// Records a key that will be deleted from the database when the method is invoked. - /// - /// The table that will be updated. - /// The table key that will be removed. - /// This interface for fluent operations. - public IDbBatch Delete(byte table, byte[] key) - { - this.cache[new byte[] { table }.Concat(key).ToArray()] = null; - return this.batch.Delete(table, key); - } - - /// - /// Returns any changes that have been made to the batch or retrieves the value from the underlying database.. - /// - /// The table of the value to be retrieved. - /// The table key of the value to retrieve. - /// This interface for fluent operations. - public byte[] Get(byte table, byte[] key) - { - if (this.cache.TryGetValue(new byte[] { table }.Concat(key).ToArray(), out byte[] value)) - return value; - - return this.db.Get(table, key); - } - - /// - /// Writes the recorded changes to the database. - /// - public void Write() - { - this.batch.Write(); - } - - public void Dispose() - { - this.batch.Dispose(); - } - } - - /// - /// Extension methods that build on the interface. - /// - public static class IDbExt - { - /// - /// Gets a . - /// - /// The database to get the batch for. - /// The . - public static ReadWriteBatch GetReadWriteBatch(this IDb db) - { - return new ReadWriteBatch(db); - } - } - /// /// An iterator that can be used to iterate the keys and values in an compliant database. /// @@ -214,109 +123,4 @@ public interface IDbIterator : IDisposable /// The value. byte[] Value(); } - - /// - /// Extension methods that build on the interface. - /// - public static class IDbIteratorExt - { - private static ByteArrayComparer byteArrayComparer = new ByteArrayComparer(); - - /// - /// Gets all the keys in the relevant table subject to any supplied constraints. - /// - /// The iterator that also identifies the table being iterated. - /// Defaults to false. Set to true if values should be ommitted - i.e. set to null. - /// Defaults to true. Set to false to return keys in ascending order. - /// Can be set optionally to specify the lower bound of keys to return. - /// Can be set optionally to specify the upper bound of keys to return. - /// Defaults to true. Set to false to omit the key specified in . - /// Defaults to true. Set to false to omit the key specified in . - /// An enumeration containing all the keys and values according to the specified constraints. - public static IEnumerable<(byte[], byte[])> GetAll(this IDbIterator iterator, bool keysOnly = false, bool ascending = true, - byte[] firstKey = null, byte[] lastKey = null, bool includeFirstKey = true, bool includeLastKey = true) - { - bool done = false; - Func breakLoop; - Action next; - - if (!ascending) - { - // Seek to the last key if it was provided. - if (lastKey == null) - iterator.SeekToLast(); - else - { - iterator.Seek(lastKey); - if (!(includeLastKey && iterator.IsValid() && byteArrayComparer.Equals(iterator.Key(), lastKey))) - iterator.Prev(); - } - - breakLoop = (firstKey == null) ? (Func)null : (keyBytes) => - { - int compareResult = byteArrayComparer.Compare(keyBytes, firstKey); - if (compareResult <= 0) - { - // If this is the first key and its not included or we've overshot the range then stop without yielding a value. - if (!includeFirstKey || compareResult < 0) - return true; - - // Stop after yielding the value. - done = true; - } - - // Keep going. - return false; - }; - - next = () => iterator.Prev(); - } - else /* Ascending */ - { - // Seek to the first key if it was provided. - if (firstKey == null) - iterator.Seek(new byte[0]); - else - { - iterator.Seek(firstKey); - if (!(includeFirstKey && iterator.IsValid() && byteArrayComparer.Equals(iterator.Key(), firstKey))) - iterator.Next(); - } - - breakLoop = (lastKey == null) ? (Func)null : (keyBytes) => - { - int compareResult = byteArrayComparer.Compare(keyBytes, lastKey); - if (compareResult >= 0) - { - // If this is the last key and its not included or we've overshot the range then stop without yielding a value. - if (!includeLastKey || compareResult > 0) - return true; - - // Stop after yielding the value. - done = true; - } - - // Keep going. - return false; - }; - - next = () => iterator.Next(); - } - - while (iterator.IsValid()) - { - byte[] keyBytes = iterator.Key(); - - if (breakLoop != null && breakLoop(keyBytes)) - break; - - yield return (keyBytes, keysOnly ? null : iterator.Value()); - - if (done) - break; - - next(); - } - } - } -} +} \ No newline at end of file From ae6663f8eece011e6b5e29fbf1466bbd16cc91a5 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 11 Jul 2022 22:11:53 +1000 Subject: [PATCH 3/5] Update db variable name --- .../CoinViews/Coindb/Coindb.cs | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs index cb536974d0..c8b596510b 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs @@ -41,7 +41,7 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews private BackendPerformanceSnapshot latestPerformanceSnapShot; /// Access to dBreeze database. - private IDb leveldb; + private IDb db; private readonly DBreezeSerializer dBreezeSerializer; @@ -72,27 +72,27 @@ public Coindb(Network network, string dataFolder, IDateTimeProvider dateTimeProv public void Initialize(ChainedHeader chainTip) { // Open a connection to a new DB and create if not found - this.leveldb = new T(); - this.leveldb.Open(this.dataFolder); + this.db = new T(); + this.db.Open(this.dataFolder); // Check if key bytes are in the wrong endian order. HashHeightPair current = this.GetTipHash(); if (current != null) { - byte[] row = this.leveldb.Get(rewindTable, BitConverter.GetBytes(current.Height)); + byte[] row = this.db.Get(rewindTable, BitConverter.GetBytes(current.Height)); // Fix the table if required. if (row != null) { // To be sure, check the next height too. - byte[] row2 = (current.Height > 1) ? this.leveldb.Get(rewindTable, BitConverter.GetBytes(current.Height - 1)) : new byte[] { }; + byte[] row2 = (current.Height > 1) ? this.db.Get(rewindTable, BitConverter.GetBytes(current.Height - 1)) : new byte[] { }; if (row2 != null) { this.logger.LogInformation("Fixing the coin db."); var rows = new Dictionary(); - using (var iterator = this.leveldb.GetIterator(rewindTable)) + using (var iterator = this.db.GetIterator(rewindTable)) { iterator.Seek(new byte[0]); @@ -111,7 +111,7 @@ public void Initialize(ChainedHeader chainTip) } } - using (var batch = this.leveldb.GetWriteBatch()) + using (var batch = this.db.GetWriteBatch()) { foreach (int height in rows.Keys.OrderBy(k => k)) { @@ -135,7 +135,7 @@ public void Initialize(ChainedHeader chainTip) if (this.GetTipHash() == null) { - using (var batch = this.leveldb.GetWriteBatch()) + using (var batch = this.db.GetWriteBatch()) { this.SetBlockHash(batch, new HashHeightPair(genesis.GetHash(), 0)); batch.Write(); @@ -156,7 +156,7 @@ private void EnsureCoinDatabaseIntegrity(ChainedHeader chainTip) { heightToCheck += 1; - byte[] row = this.leveldb.Get(rewindTable, BitConverter.GetBytes(heightToCheck).Reverse().ToArray()); + byte[] row = this.db.Get(rewindTable, BitConverter.GetBytes(heightToCheck).Reverse().ToArray()); if (row == null) break; @@ -183,7 +183,7 @@ public HashHeightPair GetTipHash() { if (this.persistedCoinviewTip == null) { - var row = this.leveldb.Get(blockTable, blockHashKey); + var row = this.db.Get(blockTable, blockHashKey); if (row != null) { this.persistedCoinviewTip = new HashHeightPair(); @@ -204,7 +204,7 @@ public FetchCoinsResponse FetchCoins(OutPoint[] utxos) foreach (OutPoint outPoint in utxos) { - byte[] row = this.leveldb.Get(coinsTable, outPoint.ToBytes()); + byte[] row = this.db.Get(coinsTable, outPoint.ToBytes()); Coins outputs = row != null ? this.dBreezeSerializer.Deserialize(row) : null; this.logger.LogDebug("Outputs for '{0}' were {1}.", outPoint, outputs == null ? "NOT loaded" : "loaded"); @@ -220,7 +220,7 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB { int insertedEntities = 0; - using (var batch = this.leveldb.GetWriteBatch()) + using (var batch = this.db.GetWriteBatch()) { using (new StopwatchDisposable(o => this.performanceCounter.AddInsertTime(o))) { @@ -282,7 +282,7 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB public int GetMinRewindHeight() { // Find the first row with a rewind table key prefix. - using (var iterator = this.leveldb.GetIterator(rewindTable)) + using (var iterator = this.db.GetIterator(rewindTable)) { iterator.Seek(new byte[0]); if (!iterator.IsValid()) @@ -308,11 +308,11 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) { HashHeightPair res = null; - using (var batch = this.leveldb.GetWriteBatch()) + using (var batch = this.db.GetWriteBatch()) { for (int height = startHeight; height > (target?.Height ?? (startHeight - 1)) && height > (startHeight - MaxRewindBatchSize); height--) { - byte[] row = this.leveldb.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); + byte[] row = this.db.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); if (row == null) throw new InvalidOperationException($"No rewind data found for block at height {height}."); @@ -345,7 +345,7 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) public RewindData GetRewindData(int height) { - byte[] row = this.leveldb.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); + byte[] row = this.db.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); return row != null ? this.dBreezeSerializer.Deserialize(row) : null; } @@ -355,7 +355,7 @@ public RewindData GetRewindData(int height) /// List of POS block information to be examined and persists if unsaved. public void PutStake(IEnumerable stakeEntries) { - using (var batch = this.leveldb.GetWriteBatch()) + using (var batch = this.db.GetWriteBatch()) { foreach (StakeItem stakeEntry in stakeEntries) { @@ -379,7 +379,7 @@ public void GetStake(IEnumerable blocklist) foreach (StakeItem blockStake in blocklist) { this.logger.LogTrace("Loading POS block hash '{0}' from the database.", blockStake.BlockId); - byte[] stakeRow = this.leveldb.Get(stakeTable, blockStake.BlockId.ToBytes(false)); + byte[] stakeRow = this.db.Get(stakeTable, blockStake.BlockId.ToBytes(false)); if (stakeRow != null) { @@ -406,7 +406,7 @@ private void AddBenchStats(StringBuilder log) /// public void Dispose() { - this.leveldb.Dispose(); + this.db.Dispose(); } } } From ec5c73a7bb37e610c60ae7e8ccb837624d555668 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Mon, 11 Jul 2022 22:42:04 +1000 Subject: [PATCH 4/5] Update db variable name --- .../CoinViews/Coindb/Coindb.cs | 54 ++++++++----------- .../CoinViews/Coindb/ICoindb.cs | 8 +++ 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs index c8b596510b..be0cd8603a 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/Coindb.cs @@ -12,8 +12,9 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews { /// - /// Persistent implementation of coinview using the dBreeze database engine. + /// Persistent implementation of coinview using an database engine. /// + /// A database supporting the interface. public class Coindb : ICoindb, IStakedb, IDisposable where T : IDb, new() { /// Database key under which the block hash of the coin view's current tip is stored. @@ -41,7 +42,7 @@ namespace Stratis.Bitcoin.Features.Consensus.CoinViews private BackendPerformanceSnapshot latestPerformanceSnapShot; /// Access to dBreeze database. - private IDb db; + private IDb coinDb; private readonly DBreezeSerializer dBreezeSerializer; @@ -72,27 +73,27 @@ public Coindb(Network network, string dataFolder, IDateTimeProvider dateTimeProv public void Initialize(ChainedHeader chainTip) { // Open a connection to a new DB and create if not found - this.db = new T(); - this.db.Open(this.dataFolder); + this.coinDb = new T(); + this.coinDb.Open(this.dataFolder); // Check if key bytes are in the wrong endian order. HashHeightPair current = this.GetTipHash(); if (current != null) { - byte[] row = this.db.Get(rewindTable, BitConverter.GetBytes(current.Height)); + byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(current.Height)); // Fix the table if required. if (row != null) { // To be sure, check the next height too. - byte[] row2 = (current.Height > 1) ? this.db.Get(rewindTable, BitConverter.GetBytes(current.Height - 1)) : new byte[] { }; + byte[] row2 = (current.Height > 1) ? this.coinDb.Get(rewindTable, BitConverter.GetBytes(current.Height - 1)) : new byte[] { }; if (row2 != null) { this.logger.LogInformation("Fixing the coin db."); var rows = new Dictionary(); - using (var iterator = this.db.GetIterator(rewindTable)) + using (var iterator = this.coinDb.GetIterator(rewindTable)) { iterator.Seek(new byte[0]); @@ -111,7 +112,7 @@ public void Initialize(ChainedHeader chainTip) } } - using (var batch = this.db.GetWriteBatch()) + using (var batch = this.coinDb.GetWriteBatch()) { foreach (int height in rows.Keys.OrderBy(k => k)) { @@ -135,7 +136,7 @@ public void Initialize(ChainedHeader chainTip) if (this.GetTipHash() == null) { - using (var batch = this.db.GetWriteBatch()) + using (var batch = this.coinDb.GetWriteBatch()) { this.SetBlockHash(batch, new HashHeightPair(genesis.GetHash(), 0)); batch.Write(); @@ -156,7 +157,7 @@ private void EnsureCoinDatabaseIntegrity(ChainedHeader chainTip) { heightToCheck += 1; - byte[] row = this.db.Get(rewindTable, BitConverter.GetBytes(heightToCheck).Reverse().ToArray()); + byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(heightToCheck).Reverse().ToArray()); if (row == null) break; @@ -183,7 +184,7 @@ public HashHeightPair GetTipHash() { if (this.persistedCoinviewTip == null) { - var row = this.db.Get(blockTable, blockHashKey); + var row = this.coinDb.Get(blockTable, blockHashKey); if (row != null) { this.persistedCoinviewTip = new HashHeightPair(); @@ -204,7 +205,7 @@ public FetchCoinsResponse FetchCoins(OutPoint[] utxos) foreach (OutPoint outPoint in utxos) { - byte[] row = this.db.Get(coinsTable, outPoint.ToBytes()); + byte[] row = this.coinDb.Get(coinsTable, outPoint.ToBytes()); Coins outputs = row != null ? this.dBreezeSerializer.Deserialize(row) : null; this.logger.LogDebug("Outputs for '{0}' were {1}.", outPoint, outputs == null ? "NOT loaded" : "loaded"); @@ -220,7 +221,7 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB { int insertedEntities = 0; - using (var batch = this.db.GetWriteBatch()) + using (var batch = this.coinDb.GetWriteBatch()) { using (new StopwatchDisposable(o => this.performanceCounter.AddInsertTime(o))) { @@ -282,7 +283,7 @@ public void SaveChanges(IList unspentOutputs, HashHeightPair oldB public int GetMinRewindHeight() { // Find the first row with a rewind table key prefix. - using (var iterator = this.db.GetIterator(rewindTable)) + using (var iterator = this.coinDb.GetIterator(rewindTable)) { iterator.Seek(new byte[0]); if (!iterator.IsValid()) @@ -297,7 +298,6 @@ public int GetMinRewindHeight() } } - /// public HashHeightPair Rewind(HashHeightPair target) { HashHeightPair current = this.GetTipHash(); @@ -308,11 +308,11 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) { HashHeightPair res = null; - using (var batch = this.db.GetWriteBatch()) + using (var batch = this.coinDb.GetWriteBatch()) { for (int height = startHeight; height > (target?.Height ?? (startHeight - 1)) && height > (startHeight - MaxRewindBatchSize); height--) { - byte[] row = this.db.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); + byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); if (row == null) throw new InvalidOperationException($"No rewind data found for block at height {height}."); @@ -345,17 +345,13 @@ private HashHeightPair RewindInternal(int startHeight, HashHeightPair target) public RewindData GetRewindData(int height) { - byte[] row = this.db.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); + byte[] row = this.coinDb.Get(rewindTable, BitConverter.GetBytes(height).Reverse().ToArray()); return row != null ? this.dBreezeSerializer.Deserialize(row) : null; } - /// - /// Persists unsaved POS blocks information to the database. - /// - /// List of POS block information to be examined and persists if unsaved. public void PutStake(IEnumerable stakeEntries) { - using (var batch = this.db.GetWriteBatch()) + using (var batch = this.coinDb.GetWriteBatch()) { foreach (StakeItem stakeEntry in stakeEntries) { @@ -370,16 +366,12 @@ public void PutStake(IEnumerable stakeEntries) } } - /// - /// Retrieves POS blocks information from the database. - /// - /// List of partially initialized POS block information that is to be fully initialized with the values from the database. public void GetStake(IEnumerable blocklist) { foreach (StakeItem blockStake in blocklist) { this.logger.LogTrace("Loading POS block hash '{0}' from the database.", blockStake.BlockId); - byte[] stakeRow = this.db.Get(stakeTable, blockStake.BlockId.ToBytes(false)); + byte[] stakeRow = this.coinDb.Get(stakeTable, blockStake.BlockId.ToBytes(false)); if (stakeRow != null) { @@ -391,7 +383,7 @@ public void GetStake(IEnumerable blocklist) private void AddBenchStats(StringBuilder log) { - log.AppendLine(">> Leveldb Bench"); + log.AppendLine(">> Coindb Bench"); BackendPerformanceSnapshot snapShot = this.performanceCounter.Snapshot(); @@ -406,7 +398,7 @@ private void AddBenchStats(StringBuilder log) /// public void Dispose() { - this.db.Dispose(); + this.coinDb.Dispose(); } } -} +} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs index 32b23b5f49..f8c664d6b6 100644 --- a/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs +++ b/src/Stratis.Bitcoin.Features.Consensus/CoinViews/Coindb/ICoindb.cs @@ -80,8 +80,16 @@ public interface ICoindb public interface IStakedb : ICoindb { + /// + /// Persists unsaved POS blocks information to the database. + /// + /// List of POS block information to be examined and persists if unsaved. void PutStake(IEnumerable stakeEntries); + /// + /// Retrieves POS blocks information from the database. + /// + /// List of partially initialized POS block information that is to be fully initialized with the values from the database. void GetStake(IEnumerable blocklist); } } From 02e8ca197918b4beb90f9678bd1a88f97c5db9c6 Mon Sep 17 00:00:00 2001 From: quantumagi Date: Tue, 12 Jul 2022 16:46:30 +1000 Subject: [PATCH 5/5] Add comments to SeekToLast --- src/Stratis.Bitcoin/Database/LevelDb.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Stratis.Bitcoin/Database/LevelDb.cs b/src/Stratis.Bitcoin/Database/LevelDb.cs index a81e78486a..0fa28a7e14 100644 --- a/src/Stratis.Bitcoin/Database/LevelDb.cs +++ b/src/Stratis.Bitcoin/Database/LevelDb.cs @@ -87,10 +87,13 @@ public void Seek(byte[] key) public void SeekToLast() { + // First seek past the last record in the table by attempting to seek to the start of the next table (if any). this.iterator.Seek(new[] { (byte)(this.table + 1) }); if (!this.iterator.IsValid()) + // If there is no next table then simply seek to the last record in the db as that will be the last record of 'table'. this.iterator.SeekToLast(); else + // If we managed to seek to the start of the next table then go back one record to arrive at the last record of 'table'. this.iterator.Prev(); }