Skip to content

Commit b5910a6

Browse files
ztlpnztlpn
committed
Datashard configurable stats report intervals (ydb-platform#26862)
(cherry picked from commit efa0852) Conflicts: ydb/core/sys_view/ut_common.h ydb/core/tx/schemeshard/ut_helpers/test_env.cpp ydb/core/tx/schemeshard/ut_helpers/test_env.h
1 parent b356229 commit b5910a6

File tree

23 files changed

+143
-65
lines changed

23 files changed

+143
-65
lines changed

‎ydb/core/client/flat_ut.cpp‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2631,7 +2631,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
26312631
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 1);
26322632

26332633
// Force stats reporting without delays
2634-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2634+
cleverServer.GetRuntime()->GetAppData()
2635+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
26352636

26362637
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
26372638
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_DEBUG);
@@ -2748,7 +2749,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
27482749
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 1);
27492750

27502751
// Force stats reporting without delays
2751-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2752+
cleverServer.GetRuntime()->GetAppData()
2753+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
27522754

27532755
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
27542756
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_DEBUG);
@@ -2921,7 +2923,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
29212923
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 4);
29222924

29232925
// Force stats reporting without delays
2924-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2926+
cleverServer.GetRuntime()->GetAppData()
2927+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
29252928

29262929
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
29272930
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_DEBUG);
@@ -3027,7 +3030,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
30273030
TVector<ui64> initialPartitions = annoyingClient.GetTablePartitions("/dc-1/Dir/T1");
30283031

30293032
// Force stats reporting without delays
3030-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3033+
cleverServer.GetRuntime()->GetAppData()
3034+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
30313035
NDataShard::gDbStatsDataSizeResolution = 80000;
30323036

30333037
TString bigValue(100*1024, '0');

‎ydb/core/protos/datashard_config.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,6 @@ message TDataShardConfig {
2626
optional uint64 IncrementalRestoreReadAheadLo = 20 [default = 524288];
2727
optional uint64 IncrementalRestoreReadAheadHi = 21 [default = 1048576];
2828
optional uint64 InMemoryStateMigrationTimeoutMs = 24 [default = 1000];
29+
optional uint32 StatsReportIntervalSeconds = 25 [default = 10];
30+
optional uint32 BackupTableStatsReportIntervalSeconds = 26 [default = 30];
2931
}

‎ydb/core/sys_view/ut_common.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,11 @@ TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, const TTestEnvSettings&
6363

6464
Settings->AppConfig->MutableHiveConfig()->AddBalancerIgnoreTabletTypes(NKikimrTabletBase::TTabletTypes::SysViewProcessor);
6565

66+
if (settings.DataShardStatsReportIntervalSeconds) {
67+
Settings->AppConfig->MutableDataShardConfig()
68+
->SetStatsReportIntervalSeconds(*settings.DataShardStatsReportIntervalSeconds);
69+
}
70+
6671
Server = new Tests::TServer(*Settings);
6772
Server->EnableGRpc(grpcPort);
6873

‎ydb/core/sys_view/ut_common.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ struct TTestEnvSettings {
2121
ui32 PqTabletsN = 0;
2222
bool EnableSVP = false;
2323
bool EnableForceFollowers = false;
24+
TMaybe<ui32> DataShardStatsReportIntervalSeconds;
2425
};
2526

2627
class TTestEnv {

‎ydb/core/sys_view/ut_kqp.cpp‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -846,9 +846,7 @@ Y_UNIT_TEST_SUITE(SystemView) {
846846
}
847847

848848
Y_UNIT_TEST(PartitionStatsLocksFields) {
849-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
850-
851-
TTestEnv env;
849+
TTestEnv env({.DataShardStatsReportIntervalSeconds = 0});
852850
CreateRootTable(env, /* partitionCount */ 1, /* fillTable */ true);
853851

854852
TTableClient client(env.GetDriver());
@@ -876,11 +874,9 @@ Y_UNIT_TEST_SUITE(SystemView) {
876874
}
877875

878876
Y_UNIT_TEST(PartitionStatsFields) {
879-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
880-
881877
auto nowUs = TInstant::Now().MicroSeconds();
882878

883-
TTestEnv env;
879+
TTestEnv env({.DataShardStatsReportIntervalSeconds = 0});
884880
CreateRootTable(env);
885881

886882
TTableClient client(env.GetDriver());
@@ -1357,11 +1353,9 @@ Y_UNIT_TEST_SUITE(SystemView) {
13571353
}
13581354

13591355
Y_UNIT_TEST(TopPartitionsByCpuFields) {
1360-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
1361-
13621356
auto nowUs = TInstant::Now().MicroSeconds();
13631357

1364-
TTestEnv env(1, 4, {.EnableSVP = true});
1358+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
13651359
CreateTenantsAndTables(env);
13661360

13671361
TTableClient client(env.GetDriver());
@@ -1409,11 +1403,9 @@ Y_UNIT_TEST_SUITE(SystemView) {
14091403
}
14101404

14111405
Y_UNIT_TEST(TopPartitionsByCpuTables) {
1412-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
1413-
14141406
constexpr ui64 partitionCount = 5;
14151407

1416-
TTestEnv env(1, 4, {.EnableSVP = true});
1408+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
14171409
CreateTenantsAndTables(env, true, partitionCount);
14181410

14191411
TTableClient client(env.GetDriver());
@@ -1439,11 +1431,9 @@ Y_UNIT_TEST_SUITE(SystemView) {
14391431
}
14401432

14411433
Y_UNIT_TEST(TopPartitionsByCpuRanges) {
1442-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
1443-
14441434
constexpr ui64 partitionCount = 5;
14451435

1446-
TTestEnv env(1, 4, {.EnableSVP = true});
1436+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
14471437
CreateTenantsAndTables(env, true, partitionCount);
14481438

14491439
TTableClient client(env.GetDriver());
@@ -1520,11 +1510,13 @@ Y_UNIT_TEST_SUITE(SystemView) {
15201510
}
15211511

15221512
Y_UNIT_TEST(TopPartitionsByCpuFollowers) {
1523-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
1524-
15251513
auto nowUs = TInstant::Now().MicroSeconds();
15261514

1527-
TTestEnv env(1, 4, {.EnableSVP = true, .EnableForceFollowers = true});
1515+
TTestEnv env(1, 4, {
1516+
.EnableSVP = true,
1517+
.EnableForceFollowers = true,
1518+
.DataShardStatsReportIntervalSeconds = 0,
1519+
});
15281520

15291521
auto& runtime = *env.GetServer().GetRuntime();
15301522
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
@@ -1695,9 +1687,7 @@ Y_UNIT_TEST_SUITE(SystemView) {
16951687
}
16961688

16971689
Y_UNIT_TEST(TopPartitionsByTliFields) {
1698-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
1699-
1700-
TTestEnv env(1, 4, {.EnableSVP = true});
1690+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
17011691
CreateTenantsAndTables(env);
17021692

17031693
TTableClient client(env.GetDriver());

‎ydb/core/testlib/test_client.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -540,6 +540,7 @@ namespace Tests {
540540
appData.DataStreamsAuthFactory = Settings->DataStreamsAuthFactory.get();
541541
appData.PersQueueMirrorReaderFactory = Settings->PersQueueMirrorReaderFactory.get();
542542
appData.HiveConfig.MergeFrom(Settings->AppConfig->GetHiveConfig());
543+
appData.DataShardConfig.MergeFrom(Settings->AppConfig->GetDataShardConfig());
543544
appData.GraphConfig.MergeFrom(Settings->AppConfig->GetGraphConfig());
544545
appData.SqsConfig.MergeFrom(Settings->AppConfig->GetSqsConfig());
545546
appData.SharedCacheConfig.MergeFrom(Settings->AppConfig->GetSharedCacheConfig());

‎ydb/core/tx/datashard/datashard.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ using namespace NTabletFlatExecutor;
3535
// But in unit tests we want to test both scenarios
3636
bool gAllowLogBatchingDefaultValue = true;
3737

38-
TDuration gDbStatsReportInterval = TDuration::Seconds(10);
3938
ui64 gDbStatsDataSizeResolution = 10*1024*1024;
4039
ui64 gDbStatsRowCountResolution = 100000;
4140
ui32 gDbStatsHistogramBucketsCount = 10;

‎ydb/core/tx/datashard/datashard.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,6 @@ namespace NDataShard {
148148

149149
// NOTE: this switch should be modified only in tests !!!
150150
extern bool gAllowLogBatchingDefaultValue;
151-
extern TDuration gDbStatsReportInterval;
152151
extern ui64 gDbStatsDataSizeResolution;
153152
extern ui64 gDbStatsRowCountResolution;
154153
extern ui32 gDbStatsHistogramBucketsCount;

‎ydb/core/tx/datashard/datashard__stats.cpp‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -597,13 +597,31 @@ class TDataShard::TTxInitiateStatsUpdate : public NTabletFlatExecutor::TTransact
597597
}
598598
};
599599

600+
TDuration TDataShard::GetStatsReportInterval(const TAppData& appData) const {
601+
const auto& userTables = GetUserTables();
602+
const bool isBackup = !userTables.empty() && std::all_of(userTables.begin(), userTables.end(),
603+
[](const auto& kv) { return kv.second->IsBackup; });
604+
605+
if (isBackup) {
606+
// Clamp the interval for backup tables to the value for ordinary tables, as it
607+
// makes no sense for the latter to be longer than the former.
608+
auto interval = std::max(
609+
appData.DataShardConfig.GetBackupTableStatsReportIntervalSeconds(),
610+
appData.DataShardConfig.GetStatsReportIntervalSeconds());
611+
return TDuration::Seconds(interval);
612+
} else {
613+
return TDuration::Seconds(appData.DataShardConfig.GetStatsReportIntervalSeconds());
614+
}
615+
}
616+
600617
void TDataShard::UpdateTableStats(const TActorContext &ctx) {
601618
if (StatisticsDisabled)
602619
return;
603620

604-
TInstant now = AppData(ctx)->TimeProvider->Now();
621+
auto* appData = AppData(ctx);
622+
TInstant now = appData->TimeProvider->Now();
605623

606-
if (LastDbStatsUpdateTime + gDbStatsReportInterval > now)
624+
if (LastDbStatsUpdateTime + GetStatsReportInterval(*appData) > now)
607625
return;
608626

609627
if (State != TShardState::Ready && State != TShardState::Readonly)

‎ydb/core/tx/datashard/datashard_impl.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3294,13 +3294,16 @@ class TDataShard
32943294
NTabletPipe::SendData(ctx, StateReportPipe, ev.Release());
32953295
}
32963296

3297+
TDuration GetStatsReportInterval(const TAppData&) const;
3298+
32973299
void SendPeriodicTableStats(const TActorContext &ctx) {
32983300
if (StatisticsDisabled)
32993301
return;
33003302

3301-
TInstant now = AppData(ctx)->TimeProvider->Now();
3303+
auto* appData = AppData(ctx);
3304+
TInstant now = appData->TimeProvider->Now();
33023305

3303-
if (LastDbStatsReportTime + gDbStatsReportInterval > now)
3306+
if (LastDbStatsReportTime + GetStatsReportInterval(*appData) > now)
33043307
return;
33053308

33063309
auto* resourceMetrics = Executor()->GetResourceMetrics();

0 commit comments

Comments
 (0)