Skip to content

Commit efa0852

Browse files
ztlpnztlpn
authored
Datashard configurable stats report intervals (#26862)
1 parent 78aa6fa commit efa0852

File tree

23 files changed

+141
-64
lines changed

23 files changed

+141
-64
lines changed

‎ydb/core/client/flat_ut.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2627,7 +2627,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
26272627
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 1);
26282628

26292629
// Force stats reporting without delays
2630-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2630+
cleverServer.GetRuntime()->GetAppData()
2631+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
26312632

26322633
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
26332634
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_DEBUG);
@@ -2800,7 +2801,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
28002801
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 4);
28012802

28022803
// Force stats reporting without delays
2803-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2804+
cleverServer.GetRuntime()->GetAppData()
2805+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
28042806

28052807
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
28062808
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_DEBUG);
@@ -2906,7 +2908,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
29062908
TVector<ui64> initialPartitions = annoyingClient.GetTablePartitions("/dc-1/Dir/T1");
29072909

29082910
// Force stats reporting without delays
2909-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2911+
cleverServer.GetRuntime()->GetAppData()
2912+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
29102913
NDataShard::gDbStatsDataSizeResolution = 80000;
29112914

29122915
TString bigValue(100*1024, '0');

‎ydb/core/protos/datashard_config.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,6 @@ message TDataShardConfig {
2626
optional uint64 IncrementalRestoreReadAheadLo = 20 [default = 524288];
2727
optional uint64 IncrementalRestoreReadAheadHi = 21 [default = 1048576];
2828
optional uint64 InMemoryStateMigrationTimeoutMs = 24 [default = 1000];
29+
optional uint32 StatsReportIntervalSeconds = 25 [default = 10];
30+
optional uint32 BackupTableStatsReportIntervalSeconds = 26 [default = 30];
2931
}

‎ydb/core/sys_view/ut_common.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,11 @@ TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, const TTestEnvSettings&
7878

7979
Settings->AppConfig->MutableHiveConfig()->AddBalancerIgnoreTabletTypes(NKikimrTabletBase::TTabletTypes::SysViewProcessor);
8080

81+
if (settings.DataShardStatsReportIntervalSeconds) {
82+
Settings->AppConfig->MutableDataShardConfig()
83+
->SetStatsReportIntervalSeconds(*settings.DataShardStatsReportIntervalSeconds);
84+
}
85+
8186
Server = new Tests::TServer(*Settings);
8287
Server->EnableGRpc(grpcPort);
8388

‎ydb/core/sys_view/ut_common.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ struct TTestEnvSettings {
2929
bool EnableTableCacheModes = false;
3030
TMaybe<bool> EnableRealSystemViewPaths;
3131
NKikimrProto::TAuthConfig AuthConfig = {};
32+
TMaybe<ui32> DataShardStatsReportIntervalSeconds;
3233
};
3334

3435
class TTestEnv {

‎ydb/core/sys_view/ut_kqp.cpp‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2834,9 +2834,7 @@ R"(CREATE TABLE `test_show_create` (
28342834
}
28352835

28362836
Y_UNIT_TEST(PartitionStatsLocksFields) {
2837-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2838-
2839-
TTestEnv env;
2837+
TTestEnv env({.DataShardStatsReportIntervalSeconds = 0});
28402838
CreateRootTable(env, /* partitionCount */ 1, /* fillTable */ true);
28412839

28422840
TTableClient client(env.GetDriver());
@@ -2864,11 +2862,9 @@ R"(CREATE TABLE `test_show_create` (
28642862
}
28652863

28662864
Y_UNIT_TEST(PartitionStatsFields) {
2867-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2868-
28692865
auto nowUs = TInstant::Now().MicroSeconds();
28702866

2871-
TTestEnv env;
2867+
TTestEnv env({.DataShardStatsReportIntervalSeconds = 0});
28722868
CreateRootTable(env);
28732869
const auto describeResult = env.GetClient().Describe(env.GetServer().GetRuntime(), "Root/Table0");
28742870
const auto tablePathId = describeResult.GetPathId();
@@ -3359,11 +3355,9 @@ R"(CREATE TABLE `test_show_create` (
33593355
}
33603356

33613357
Y_UNIT_TEST(TopPartitionsByCpuFields) {
3362-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3363-
33643358
auto nowUs = TInstant::Now().MicroSeconds();
33653359

3366-
TTestEnv env(1, 4, {.EnableSVP = true});
3360+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
33673361
CreateTenantsAndTables(env);
33683362

33693363
TTableClient client(env.GetDriver());
@@ -3411,11 +3405,9 @@ R"(CREATE TABLE `test_show_create` (
34113405
}
34123406

34133407
Y_UNIT_TEST(TopPartitionsByCpuTables) {
3414-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3415-
34163408
constexpr ui64 partitionCount = 5;
34173409

3418-
TTestEnv env(1, 4, {.EnableSVP = true});
3410+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
34193411
CreateTenantsAndTables(env, true, partitionCount);
34203412

34213413
TTableClient client(env.GetDriver());
@@ -3441,11 +3433,9 @@ R"(CREATE TABLE `test_show_create` (
34413433
}
34423434

34433435
Y_UNIT_TEST(TopPartitionsByCpuRanges) {
3444-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3445-
34463436
constexpr ui64 partitionCount = 5;
34473437

3448-
TTestEnv env(1, 4, {.EnableSVP = true});
3438+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
34493439
CreateTenantsAndTables(env, true, partitionCount);
34503440

34513441
TTableClient client(env.GetDriver());
@@ -3522,11 +3512,13 @@ R"(CREATE TABLE `test_show_create` (
35223512
}
35233513

35243514
Y_UNIT_TEST(TopPartitionsByCpuFollowers) {
3525-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3526-
35273515
auto nowUs = TInstant::Now().MicroSeconds();
35283516

3529-
TTestEnv env(1, 4, {.EnableSVP = true, .EnableForceFollowers = true});
3517+
TTestEnv env(1, 4, {
3518+
.EnableSVP = true,
3519+
.EnableForceFollowers = true,
3520+
.DataShardStatsReportIntervalSeconds = 0,
3521+
});
35303522

35313523
auto& runtime = *env.GetServer().GetRuntime();
35323524
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
@@ -3697,9 +3689,7 @@ R"(CREATE TABLE `test_show_create` (
36973689
}
36983690

36993691
Y_UNIT_TEST(TopPartitionsByTliFields) {
3700-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3701-
3702-
TTestEnv env(1, 4, {.EnableSVP = true});
3692+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
37033693
CreateTenantsAndTables(env);
37043694

37053695
TTableClient client(env.GetDriver());

‎ydb/core/testlib/test_client.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -555,6 +555,7 @@ namespace Tests {
555555
appData.DataStreamsAuthFactory = Settings->DataStreamsAuthFactory.get();
556556
appData.PersQueueMirrorReaderFactory = Settings->PersQueueMirrorReaderFactory.get();
557557
appData.HiveConfig.MergeFrom(Settings->AppConfig->GetHiveConfig());
558+
appData.DataShardConfig.MergeFrom(Settings->AppConfig->GetDataShardConfig());
558559
appData.GraphConfig.MergeFrom(Settings->AppConfig->GetGraphConfig());
559560
appData.SqsConfig.MergeFrom(Settings->AppConfig->GetSqsConfig());
560561
appData.SharedCacheConfig.MergeFrom(Settings->AppConfig->GetSharedCacheConfig());

‎ydb/core/tx/datashard/datashard.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ using namespace NTabletFlatExecutor;
3535
// But in unit tests we want to test both scenarios
3636
bool gAllowLogBatchingDefaultValue = true;
3737

38-
TDuration gDbStatsReportInterval = TDuration::Seconds(10);
3938
ui64 gDbStatsDataSizeResolution = 10*1024*1024;
4039
ui64 gDbStatsRowCountResolution = 100000;
4140
ui32 gDbStatsHistogramBucketsCount = 10;

‎ydb/core/tx/datashard/datashard.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,6 @@ namespace NDataShard {
153153

154154
// NOTE: this switch should be modified only in tests !!!
155155
extern bool gAllowLogBatchingDefaultValue;
156-
extern TDuration gDbStatsReportInterval;
157156
extern ui64 gDbStatsDataSizeResolution;
158157
extern ui64 gDbStatsRowCountResolution;
159158
extern ui32 gDbStatsHistogramBucketsCount;

‎ydb/core/tx/datashard/datashard__stats.cpp‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -578,13 +578,31 @@ class TDataShard::TTxInitiateStatsUpdate : public NTabletFlatExecutor::TTransact
578578
}
579579
};
580580

581+
TDuration TDataShard::GetStatsReportInterval(const TAppData& appData) const {
582+
const auto& userTables = GetUserTables();
583+
const bool isBackup = !userTables.empty() && std::all_of(userTables.begin(), userTables.end(),
584+
[](const auto& kv) { return kv.second->IsBackup; });
585+
586+
if (isBackup) {
587+
// Clamp the interval for backup tables to the value for ordinary tables, as it
588+
// makes no sense for the latter to be longer than the former.
589+
auto interval = std::max(
590+
appData.DataShardConfig.GetBackupTableStatsReportIntervalSeconds(),
591+
appData.DataShardConfig.GetStatsReportIntervalSeconds());
592+
return TDuration::Seconds(interval);
593+
} else {
594+
return TDuration::Seconds(appData.DataShardConfig.GetStatsReportIntervalSeconds());
595+
}
596+
}
597+
581598
void TDataShard::UpdateTableStats(const TActorContext &ctx) {
582599
if (StatisticsDisabled)
583600
return;
584601

585-
TInstant now = AppData(ctx)->TimeProvider->Now();
602+
auto* appData = AppData(ctx);
603+
TInstant now = appData->TimeProvider->Now();
586604

587-
if (LastDbStatsUpdateTime + gDbStatsReportInterval > now)
605+
if (LastDbStatsUpdateTime + GetStatsReportInterval(*appData) > now)
588606
return;
589607

590608
if (State != TShardState::Ready && State != TShardState::Readonly)

‎ydb/core/tx/datashard/datashard_impl.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3348,13 +3348,16 @@ class TDataShard
33483348
NTabletPipe::SendData(ctx, StateReportPipe, ev.Release());
33493349
}
33503350

3351+
TDuration GetStatsReportInterval(const TAppData&) const;
3352+
33513353
void SendPeriodicTableStats(const TActorContext &ctx) {
33523354
if (StatisticsDisabled)
33533355
return;
33543356

3355-
TInstant now = AppData(ctx)->TimeProvider->Now();
3357+
auto* appData = AppData(ctx);
3358+
TInstant now = appData->TimeProvider->Now();
33563359

3357-
if (LastDbStatsReportTime + gDbStatsReportInterval > now)
3360+
if (LastDbStatsReportTime + GetStatsReportInterval(*appData) > now)
33583361
return;
33593362

33603363
auto* resourceMetrics = Executor()->GetResourceMetrics();

0 commit comments

Comments
 (0)