Skip to content

Commit

Permalink
24-3 Obtain UsedTableMemory from private cache stats, include histori…
Browse files Browse the repository at this point in the history 
…c indexes size (ydb-platform#6400)
  • Loading branch information
@kunga
kunga authored Jul 9, 2024
1 parent 56b5d9a commit 64fa639
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 33 deletions.
18 changes: 9 additions & 9 deletions ydb/core/tablet_flat/flat_executor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3556,23 +3556,23 @@ void TExecutor::Handle(NOps::TEvResult *ops, TProdCompact *msg, bool cancelled)
}

void TExecutor::UpdateUsedTabletMemory() {
UsedTabletMemory = 0;
// Estimate memory usage for internal executor structures.
UsedTabletMemory += 50 << 10; // 50kb
// Count the number of bytes exclusive to private cache.
// Estimate memory usage for internal executor structures:
UsedTabletMemory = 50 << 10; // 50kb

// Count the number of bytes kept in private cache (can't be offloaded right now):
if (PrivatePageCache) {
UsedTabletMemory += PrivatePageCache->GetStats().TotalExclusive;
UsedTabletMemory += PrivatePageCache->GetStats().TotalPinnedBody;
UsedTabletMemory += PrivatePageCache->GetStats().PinnedLoadSize;
}
// Estimate memory used by database structures.

// Estimate memory used by internal database structures:
auto &counters = Database->Counters();
UsedTabletMemory += counters.MemTableWaste;
UsedTabletMemory += counters.MemTableBytes;
UsedTabletMemory += counters.Parts.IndexBytes;
UsedTabletMemory += counters.Parts.OtherBytes;
UsedTabletMemory += counters.Parts.ByKeyBytes;
UsedTabletMemory += Stats->PacksMetaBytes;

// Add tablet memory usage.
// Add tablet memory usage:
UsedTabletMemory += Owner->GetMemoryUsage();
}

Expand Down
39 changes: 39 additions & 0 deletions ydb/core/tablet_flat/flat_executor_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6307,6 +6307,45 @@ Y_UNIT_TEST_SUITE(TFlatTableExecutorBTreeIndex) {
UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 330);
}

Y_UNIT_TEST(EnableLocalDBBtreeIndex_False) { // uses flat index
TMyEnvBase env;
TRowsModel rows;

auto &appData = env->GetAppData();

appData.FeatureFlags.SetEnableLocalDBBtreeIndex(false);
auto counters = MakeIntrusive<TSharedPageCacheCounters>(env->GetDynamicCounters());
int readRows = 0, failedAttempts = 0;

env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp));

auto policy = MakeIntrusive<TCompactionPolicy>();
policy->MinBTreeIndexNodeSize = 128;
env.SendSync(rows.MakeScheme(std::move(policy)));

env.SendSync(rows.VersionTo(TRowVersion(1, 10)).RowTo(0).MakeRows(1000, 950));
env.SendSync(rows.VersionTo(TRowVersion(2, 20)).RowTo(0).MakeRows(1000, 950));

env.SendSync(new NFake::TEvCompact(TRowsModel::TableId));
env.WaitFor<NFake::TEvCompacted>();

// all pages are always kept in shared cache
UNIT_ASSERT_VALUES_EQUAL(counters->ActivePages->Val(), 290);

env.SendSync(new NFake::TEvExecute{ new TTxFullScan(readRows, failedAttempts) });
UNIT_ASSERT_VALUES_EQUAL(readRows, 1000);
UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 0);

// restart tablet
env.SendSync(new TEvents::TEvPoison, false, true);
env.FireDummyTablet(ui32(NFake::TDummy::EFlg::Comp));

// after restart we have no pages in private cache
env.SendSync(new NFake::TEvExecute{ new TTxFullScan(readRows, failedAttempts) }, true);
UNIT_ASSERT_VALUES_EQUAL(readRows, 1000);
UNIT_ASSERT_VALUES_EQUAL(failedAttempts, 288);
}

Y_UNIT_TEST(EnableLocalDBBtreeIndex_True_EnableLocalDBFlatIndex_False) { // uses b-tree index
TMyEnvBase env;
TRowsModel rows;
Expand Down
7 changes: 6 additions & 1 deletion ydb/core/tablet_flat/flat_part_loader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,17 +210,22 @@ TAutoPtr<NPageCollection::TFetch> TLoader::StageCreatePartView() noexcept
TEpoch epoch = Epoch != TEpoch::Max() ? Epoch : TEpoch(Root.GetEpoch());

// TODO: put index size to stat?
// TODO: include history indexes bytes
size_t indexesRawSize = 0;
if (BTreeGroupIndexes) {
for (const auto &meta : BTreeGroupIndexes) {
indexesRawSize += meta.IndexSize;
}
for (const auto &meta : BTreeHistoricIndexes) {
indexesRawSize += meta.IndexSize;
}
// Note: although we also have flat index, it shouldn't be loaded; so let's not count it here
} else {
for (auto indexPage : FlatGroupIndexes) {
indexesRawSize += Packs[0]->GetPageSize(indexPage);
}
for (auto indexPage : FlatHistoricIndexes) {
indexesRawSize += Packs[0]->GetPageSize(indexPage);
}
}

auto *partStore = new TPartStore(
Expand Down
4 changes: 2 additions & 2 deletions ydb/core/tablet_flat/test/libs/table/test_store.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ namespace NTest {

struct TEggs {
bool Rooted;
TVector<TPageId> GroupIndexes;
TVector<TPageId> HistoricIndexes;
TVector<TPageId> FlatGroupIndexes;
TVector<TPageId> FlatHistoricIndexes;
TVector<NPage::TBtreeIndexMeta> BTreeGroupIndexes;
TVector<NPage::TBtreeIndexMeta> BTreeHistoricIndexes;
TData *Scheme;
Expand Down
10 changes: 8 additions & 2 deletions ydb/core/tablet_flat/test/libs/table/test_writer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,14 @@ namespace NTest {
for (const auto &meta : eggs.BTreeGroupIndexes) {
indexesRawSize += meta.IndexSize;
}
for (const auto &meta : eggs.BTreeHistoricIndexes) {
indexesRawSize += meta.IndexSize;
}
} else {
for (auto indexPage : eggs.GroupIndexes) {
for (auto indexPage : eggs.FlatGroupIndexes) {
indexesRawSize += Store->GetPageSize(0, indexPage);
}
for (auto indexPage : eggs.FlatHistoricIndexes) {
indexesRawSize += Store->GetPageSize(0, indexPage);
}
}
Expand All @@ -90,7 +96,7 @@ namespace NTest {
{
epoch,
TPartScheme::Parse(*eggs.Scheme, eggs.Rooted),
{ eggs.GroupIndexes, eggs.HistoricIndexes, eggs.BTreeGroupIndexes, eggs.BTreeHistoricIndexes },
{ eggs.FlatGroupIndexes, eggs.FlatHistoricIndexes, eggs.BTreeGroupIndexes, eggs.BTreeHistoricIndexes },
eggs.Blobs ? new TExtBlobs(*eggs.Blobs, { }) : nullptr,
eggs.ByKey ? new TBloom(*eggs.ByKey) : nullptr,
eggs.Large ? new TFrames(*eggs.Large) : nullptr,
Expand Down
38 changes: 19 additions & 19 deletions ydb/core/tablet_flat/ut/ut_stat.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,14 +143,14 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) {
Y_UNIT_TEST(Single_History)
{
auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
CheckMixedIndex(*subset, 24000, 3547100, 31242);
CheckMixedIndex(*subset, 24000, 3547100, 49916);
}

Y_UNIT_TEST(Single_History_Slices)
{
auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
CheckMixedIndex(*subset, 9582, 1425198, 31242);
CheckMixedIndex(*subset, 9582, 1425198, 49916);
}

Y_UNIT_TEST(Single_Groups)
Expand All @@ -169,14 +169,14 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) {
Y_UNIT_TEST(Single_Groups_History)
{
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
CheckMixedIndex(*subset, 24000, 4054050, 18810);
CheckMixedIndex(*subset, 24000, 4054050, 29361);
}

Y_UNIT_TEST(Single_Groups_History_Slices)
{
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
CheckMixedIndex(*subset, 13570, 2277890, 18810);
CheckMixedIndex(*subset, 13570, 2277890, 29361);
}

Y_UNIT_TEST(Mixed)
Expand All @@ -194,7 +194,7 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) {
Y_UNIT_TEST(Mixed_Groups_History)
{
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, TMixerRnd(4), 0.3);
CheckMixedIndex(*subset, 24000, 4054270, 19152);
CheckMixedIndex(*subset, 24000, 4054270, 29970);
}

Y_UNIT_TEST(Serial)
Expand All @@ -215,7 +215,7 @@ Y_UNIT_TEST_SUITE(BuildStatsFlatIndex) {
{
TMixerSeq mixer(4, Mass1.Saved.Size());
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, mixer, 0.3);
CheckMixedIndex(*subset, 24000, 4054290, 19168);
CheckMixedIndex(*subset, 24000, 4054290, 30013);
}
}

Expand All @@ -239,14 +239,14 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
Y_UNIT_TEST(Single_History)
{
auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
CheckMixedIndex(*subset, 24000, 3547100, 61162);
CheckMixedIndex(*subset, 24000, 3547100, 81694);
}

Y_UNIT_TEST(Single_History_Slices)
{
auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
CheckMixedIndex(*subset, 9582, 1425198, 61162);
CheckMixedIndex(*subset, 9582, 1425198, 81694);
}

Y_UNIT_TEST(Single_Groups)
Expand All @@ -265,14 +265,14 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
Y_UNIT_TEST(Single_Groups_History)
{
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
CheckMixedIndex(*subset, 24000, 4054050, 34837);
CheckMixedIndex(*subset, 24000, 4054050, 46562);
}

Y_UNIT_TEST(Single_Groups_History_Slices)
{
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
CheckMixedIndex(*subset, 13570, 2277890, 34837);
CheckMixedIndex(*subset, 13570, 2277890, 46562);
}

Y_UNIT_TEST(Mixed)
Expand All @@ -290,7 +290,7 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
Y_UNIT_TEST(Mixed_Groups_History)
{
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, TMixerRnd(4), 0.3);
CheckMixedIndex(*subset, 24000, 4054270, 34579);
CheckMixedIndex(*subset, 24000, 4054270, 46543);
}

Y_UNIT_TEST(Serial)
Expand All @@ -311,7 +311,7 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
{
TMixerSeq mixer(4, Mass1.Saved.Size());
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, mixer, 0.3);
CheckMixedIndex(*subset, 24000, 4054290, 34652);
CheckMixedIndex(*subset, 24000, 4054290, 46640);
}

Y_UNIT_TEST(Single_LowResolution)
Expand Down Expand Up @@ -343,14 +343,14 @@ Y_UNIT_TEST_SUITE(BuildStatsMixedIndex) {
Y_UNIT_TEST(Single_Groups_History_LowResolution)
{
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), true, WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
CheckMixedIndex(*subset, 24000, 4054050, 48540, 5310, 531050);
CheckMixedIndex(*subset, 24000, 4054050, 64742, 5310, 531050);
}

Y_UNIT_TEST(Single_Groups_History_Slices_LowResolution)
{
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), true, WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
CheckMixedIndex(*subset, 13570, 2234982 /* ~2277890 */, 48540, 5310, 531050);
CheckMixedIndex(*subset, 13570, 2234982 /* ~2277890 */, 64742, 5310, 531050);
}
}

Expand All @@ -374,14 +374,14 @@ Y_UNIT_TEST_SUITE(BuildStatsBTreeIndex) {
Y_UNIT_TEST(Single_History)
{
auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
CheckBTreeIndex(*subset, 24000, 3547100, 61162);
CheckBTreeIndex(*subset, 24000, 3547100, 81694);
}

Y_UNIT_TEST(Single_History_Slices)
{
auto subset = TMake(Mass0, PageConf(Mass0.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
CheckBTreeIndex(*subset, 9582, 1425282, 61162);
CheckBTreeIndex(*subset, 9582, 1425282, 81694);
}

Y_UNIT_TEST(Single_Groups)
Expand All @@ -400,14 +400,14 @@ Y_UNIT_TEST_SUITE(BuildStatsBTreeIndex) {
Y_UNIT_TEST(Single_Groups_History)
{
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3);
CheckBTreeIndex(*subset, 24000, 4054050, 34837);
CheckBTreeIndex(*subset, 24000, 4054050, 46562);
}

Y_UNIT_TEST(Single_Groups_History_Slices)
{
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 1, TMixerOne{ }, 0.3, 13);
subset->Flatten.begin()->Slices->Describe(Cerr); Cerr << Endl;
CheckBTreeIndex(*subset, 13570, 2273213, 34837);
CheckBTreeIndex(*subset, 13570, 2273213, 46562);
}

Y_UNIT_TEST(Mixed)
Expand All @@ -425,7 +425,7 @@ Y_UNIT_TEST_SUITE(BuildStatsBTreeIndex) {
Y_UNIT_TEST(Mixed_Groups_History)
{
auto subset = TMake(Mass1, PageConf(Mass1.Model->Scheme->Families.size(), WriteBTreeIndex)).Mixed(0, 4, TMixerRnd(4), 0.3);
CheckBTreeIndex(*subset, 24000, 4054270, 34579);
CheckBTreeIndex(*subset, 24000, 4054270, 46543);
}
}

Expand Down

0 comments on commit 64fa639

Please sign in to comment.