diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index 8d73dd4cc04a..81bd63b86c72 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -3556,23 +3556,23 @@ void TExecutor::Handle(NOps::TEvResult *ops, TProdCompact *msg, bool cancelled) } void TExecutor::UpdateUsedTabletMemory() { - UsedTabletMemory = 0; - // Estimate memory usage for internal executor structures. - UsedTabletMemory += 50 << 10; // 50kb - // Count the number of bytes exclusive to private cache. + // Estimate memory usage for internal executor structures: + UsedTabletMemory = 50 << 10; // 50kb + + // Count the number of bytes kept in private cache (can't be offloaded right now): if (PrivatePageCache) { - UsedTabletMemory += PrivatePageCache->GetStats().TotalExclusive; + UsedTabletMemory += PrivatePageCache->GetStats().TotalPinnedBody; + UsedTabletMemory += PrivatePageCache->GetStats().PinnedLoadSize; } - // Estimate memory used by database structures. + + // Estimate memory used by internal database structures: auto &counters = Database->Counters(); UsedTabletMemory += counters.MemTableWaste; UsedTabletMemory += counters.MemTableBytes; - UsedTabletMemory += counters.Parts.IndexBytes; UsedTabletMemory += counters.Parts.OtherBytes; - UsedTabletMemory += counters.Parts.ByKeyBytes; UsedTabletMemory += Stats->PacksMetaBytes; - // Add tablet memory usage. + // Add tablet memory usage: UsedTabletMemory += Owner->GetMemoryUsage(); } diff --git a/ydb/core/tablet_flat/flat_executor_ut.cpp b/ydb/core/tablet_flat/flat_executor_ut.cpp index 64e6db8b01d9..7583f10e790e 100644 --- a/ydb/core/tablet_flat/flat_executor_ut.cpp +++ b/ydb/core/tablet_flat/flat_executor_ut.cpp @@ -6307,6 +6307,45 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutorBTreeIndex) { UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 330); } + Y_UNIT_TEST(EnableLocalDBBtreeIndex_False) { // uses flat index + TMyEnvBase env; + TRowsModel rows; + + auto &appData = env->GetAppData(); + + appData.FeatureFlags.SetEnableLocalDBBtreeIndex(false); + auto counters = MakeIntrusive(env->GetDynamicCounters()); + int readRows = 0, failedAttempts = 0; + + env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp)); + + auto policy = MakeIntrusive(); + policy->MinBTreeIndexNodeSize = 128; + env.SendSync(rows.MakeScheme(std::move(policy))); + + env.SendSync(rows.VersionTo(TRowVersion(1, 10)).RowTo(0).MakeRows(1000, 950)); + env.SendSync(rows.VersionTo(TRowVersion(2, 20)).RowTo(0).MakeRows(1000, 950)); + + env.SendSync(new NFake::TEvCompact(TRowsModel::TableId)); + env.WaitFor(); + + // all pages are always kept in shared cache + UNIT_ASSERT_VALUES_EQUAL(counters->ActivePages->Val(), 290); + + env.SendSync(new NFake::TEvExecute{ new TTxFullScan(readRows, failedAttempts) }); + UNIT_ASSERT_VALUES_EQUAL(readRows, 1000); + UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 0); + + // restart tablet + env.SendSync(new TEvents::TEvPoison, false, true); + env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp)); + + // after restart we have no pages in private cache + env.SendSync(new NFake::TEvExecute{ new TTxFullScan(readRows, failedAttempts) }, true); + UNIT_ASSERT_VALUES_EQUAL(readRows, 1000); + UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 288); + } + Y_UNIT_TEST(EnableLocalDBBtreeIndex_True_EnableLocalDBFlatIndex_False) { // uses b-tree index TMyEnvBase env; TRowsModel rows; diff --git a/ydb/core/tablet_flat/flat_part_loader.cpp b/ydb/core/tablet_flat/flat_part_loader.cpp index b031c698cce2..33c69b131c87 100644 --- a/ydb/core/tablet_flat/flat_part_loader.cpp +++ b/ydb/core/tablet_flat/flat_part_loader.cpp @@ -210,17 +210,22 @@ TAutoPtr TLoader::StageCreatePartView() noexcept TEpoch epoch = Epoch != TEpoch::Max() ? Epoch : TEpoch(Root.GetEpoch()); // TODO: put index size to stat? - // TODO: include history indexes bytes size_t indexesRawSize = 0; if (BTreeGroupIndexes) { for (const auto &meta : BTreeGroupIndexes) { indexesRawSize += meta.IndexSize; } + for (const auto &meta : BTreeHistoricIndexes) { + indexesRawSize += meta.IndexSize; + } // Note: although we also have flat index, it shouldn't be loaded; so let's not count it here } else { for (auto indexPage : FlatGroupIndexes) { indexesRawSize += Packs[0]->GetPageSize(indexPage); } + for (auto indexPage : FlatHistoricIndexes) { + indexesRawSize += Packs[0]->GetPageSize(indexPage); + } } auto *partStore = new TPartStore( diff --git a/ydb/core/tablet_flat/test/libs/table/test_store.h b/ydb/core/tablet_flat/test/libs/table/test_store.h index e699fa92e811..761d5c939cbf 100644 --- a/ydb/core/tablet_flat/test/libs/table/test_store.h +++ b/ydb/core/tablet_flat/test/libs/table/test_store.h @@ -24,8 +24,8 @@ namespace NTest { struct TEggs { bool Rooted; - TVector GroupIndexes; - TVector HistoricIndexes; + TVector FlatGroupIndexes; + TVector FlatHistoricIndexes; TVector BTreeGroupIndexes; TVector BTreeHistoricIndexes; TData *Scheme; diff --git a/ydb/core/tablet_flat/test/libs/table/test_writer.h b/ydb/core/tablet_flat/test/libs/table/test_writer.h index 70e916d7b65c..f710071fb6c9 100644 --- a/ydb/core/tablet_flat/test/libs/table/test_writer.h +++ b/ydb/core/tablet_flat/test/libs/table/test_writer.h @@ -77,8 +77,14 @@ namespace NTest { for (const auto &meta : eggs.BTreeGroupIndexes) { indexesRawSize += meta.IndexSize; } + for (const auto &meta : eggs.BTreeHistoricIndexes) { + indexesRawSize += meta.IndexSize; + } } else { - for (auto indexPage : eggs.GroupIndexes) { + for (auto indexPage : eggs.FlatGroupIndexes) { + indexesRawSize += Store->GetPageSize(0, indexPage); + } + for (auto indexPage : eggs.FlatHistoricIndexes) { indexesRawSize += Store->GetPageSize(0, indexPage); } } @@ -90,7 +96,7 @@ namespace NTest { { epoch, TPartScheme::Parse(*eggs.Scheme, eggs.Rooted), - { eggs.GroupIndexes, eggs.HistoricIndexes, eggs.BTreeGroupIndexes, eggs.BTreeHistoricIndexes }, + { eggs.FlatGroupIndexes, eggs.FlatHistoricIndexes, eggs.BTreeGroupIndexes, eggs.BTreeHistoricIndexes }, eggs.Blobs ? new TExtBlobs(*eggs.Blobs, { }) : nullptr, eggs.ByKey ? new TBloom(*eggs.ByKey) : nullptr, eggs.Large ? new TFrames(*eggs.Large) : nullptr, diff --git a/ydb/core/tablet_flat/ut/ut_stat.cpp b/ydb/core/tablet_flat/ut/ut_stat.cpp index f17e1bfea9b9..ec265efe8daa 100644 --- a/ydb/core/tablet_flat/ut/ut_stat.cpp +++ b/ydb/core/tablet_flat/ut/ut_stat.cpp @@ -143,14 +143,14 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) { Y_UNIT_TEST(Single_History) { auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3); - CheckMixedIndex(*subset, 24000, 3547100, 31242); + CheckMixedIndex(*subset, 24000, 3547100, 49916); } Y_UNIT_TEST(Single_History_Slices) { auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13); subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl; - CheckMixedIndex(*subset, 9582, 1425198, 31242); + CheckMixedIndex(*subset, 9582, 1425198, 49916); } Y_UNIT_TEST(Single_Groups) @@ -169,14 +169,14 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) { Y_UNIT_TEST(Single_Groups_History) { auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3); - CheckMixedIndex(*subset, 24000, 4054050, 18810); + CheckMixedIndex(*subset, 24000, 4054050, 29361); } Y_UNIT_TEST(Single_Groups_History_Slices) { auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13); subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl; - CheckMixedIndex(*subset, 13570, 2277890, 18810); + CheckMixedIndex(*subset, 13570, 2277890, 29361); } Y_UNIT_TEST(Mixed) @@ -194,7 +194,7 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) { Y_UNIT_TEST(Mixed_Groups_History) { auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, TMixerRnd(4), 0.3); - CheckMixedIndex(*subset, 24000, 4054270, 19152); + CheckMixedIndex(*subset, 24000, 4054270, 29970); } Y_UNIT_TEST(Serial) @@ -215,7 +215,7 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) { { TMixerSeq mixer(4, Mass1.Saved.Size()); auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, mixer, 0.3); - CheckMixedIndex(*subset, 24000, 4054290, 19168); + CheckMixedIndex(*subset, 24000, 4054290, 30013); } } @@ -239,14 +239,14 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) { Y_UNIT_TEST(Single_History) { auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3); - CheckMixedIndex(*subset, 24000, 3547100, 61162); + CheckMixedIndex(*subset, 24000, 3547100, 81694); } Y_UNIT_TEST(Single_History_Slices) { auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13); subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl; - CheckMixedIndex(*subset, 9582, 1425198, 61162); + CheckMixedIndex(*subset, 9582, 1425198, 81694); } Y_UNIT_TEST(Single_Groups) @@ -265,14 +265,14 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) { Y_UNIT_TEST(Single_Groups_History) { auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3); - CheckMixedIndex(*subset, 24000, 4054050, 34837); + CheckMixedIndex(*subset, 24000, 4054050, 46562); } Y_UNIT_TEST(Single_Groups_History_Slices) { auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13); subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl; - CheckMixedIndex(*subset, 13570, 2277890, 34837); + CheckMixedIndex(*subset, 13570, 2277890, 46562); } Y_UNIT_TEST(Mixed) @@ -290,7 +290,7 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) { Y_UNIT_TEST(Mixed_Groups_History) { auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, TMixerRnd(4), 0.3); - CheckMixedIndex(*subset, 24000, 4054270, 34579); + CheckMixedIndex(*subset, 24000, 4054270, 46543); } Y_UNIT_TEST(Serial) @@ -311,7 +311,7 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) { { TMixerSeq mixer(4, Mass1.Saved.Size()); auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, mixer, 0.3); - CheckMixedIndex(*subset, 24000, 4054290, 34652); + CheckMixedIndex(*subset, 24000, 4054290, 46640); } Y_UNIT_TEST(Single_LowResolution) @@ -343,14 +343,14 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) { Y_UNIT_TEST(Single_Groups_History_LowResolution) { auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), true, WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3); - CheckMixedIndex(*subset, 24000, 4054050, 48540, 5310, 531050); + CheckMixedIndex(*subset, 24000, 4054050, 64742, 5310, 531050); } Y_UNIT_TEST(Single_Groups_History_Slices_LowResolution) { auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), true, WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13); subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl; - CheckMixedIndex(*subset, 13570, 2234982 /* ~2277890 */, 48540, 5310, 531050); + CheckMixedIndex(*subset, 13570, 2234982 /* ~2277890 */, 64742, 5310, 531050); } } @@ -374,14 +374,14 @@ Y_UNIT_TEST_SUITE(BuildStatsBTreeIndex) { Y_UNIT_TEST(Single_History) { auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3); - CheckBTreeIndex(*subset, 24000, 3547100, 61162); + CheckBTreeIndex(*subset, 24000, 3547100, 81694); } Y_UNIT_TEST(Single_History_Slices) { auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13); subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl; - CheckBTreeIndex(*subset, 9582, 1425282, 61162); + CheckBTreeIndex(*subset, 9582, 1425282, 81694); } Y_UNIT_TEST(Single_Groups) @@ -400,14 +400,14 @@ Y_UNIT_TEST_SUITE(BuildStatsBTreeIndex) { Y_UNIT_TEST(Single_Groups_History) { auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3); - CheckBTreeIndex(*subset, 24000, 4054050, 34837); + CheckBTreeIndex(*subset, 24000, 4054050, 46562); } Y_UNIT_TEST(Single_Groups_History_Slices) { auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13); subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl; - CheckBTreeIndex(*subset, 13570, 2273213, 34837); + CheckBTreeIndex(*subset, 13570, 2273213, 46562); } Y_UNIT_TEST(Mixed) @@ -425,7 +425,7 @@ Y_UNIT_TEST_SUITE(BuildStatsBTreeIndex) { Y_UNIT_TEST(Mixed_Groups_History) { auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, TMixerRnd(4), 0.3); - CheckBTreeIndex(*subset, 24000, 4054270, 34579); + CheckBTreeIndex(*subset, 24000, 4054270, 46543); } }