Skip to content

Commit ce2b0c3

Browse files
ztlpnztlpn
committed
datashard: configurable stats report interval
KIKIMR-24154
1 parent ff2b01d commit ce2b0c3

File tree

21 files changed

+82
-62
lines changed

21 files changed

+82
-62
lines changed

‎ydb/core/client/flat_ut.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2627,7 +2627,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
26272627
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 1);
26282628

26292629
// Force stats reporting without delays
2630-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2630+
cleverServer.GetRuntime()->GetAppData()
2631+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
26312632

26322633
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
26332634
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_DEBUG);
@@ -2800,7 +2801,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
28002801
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 4);
28012802

28022803
// Force stats reporting without delays
2803-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2804+
cleverServer.GetRuntime()->GetAppData()
2805+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
28042806

28052807
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
28062808
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_DEBUG);
@@ -2906,7 +2908,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
29062908
TVector<ui64> initialPartitions = annoyingClient.GetTablePartitions("/dc-1/Dir/T1");
29072909

29082910
// Force stats reporting without delays
2909-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2911+
cleverServer.GetRuntime()->GetAppData()
2912+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
29102913
NDataShard::gDbStatsDataSizeResolution = 80000;
29112914

29122915
TString bigValue(100*1024, '0');

‎ydb/core/protos/datashard_config.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,5 @@ message TDataShardConfig {
2626
optional uint64 IncrementalRestoreReadAheadLo = 20 [default = 524288];
2727
optional uint64 IncrementalRestoreReadAheadHi = 21 [default = 1048576];
2828
optional uint64 InMemoryStateMigrationTimeoutMs = 24 [default = 1000];
29+
optional uint32 StatsReportIntervalSeconds = 25 [default = 10];
2930
}

‎ydb/core/sys_view/ut_common.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,11 @@ TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, const TTestEnvSettings&
7878

7979
Settings->AppConfig->MutableHiveConfig()->AddBalancerIgnoreTabletTypes(NKikimrTabletBase::TTabletTypes::SysViewProcessor);
8080

81+
if (settings.DataShardStatsReportIntervalSeconds) {
82+
Settings->AppConfig->MutableDataShardConfig()
83+
->SetStatsReportIntervalSeconds(*settings.DataShardStatsReportIntervalSeconds);
84+
}
85+
8186
Server = new Tests::TServer(*Settings);
8287
Server->EnableGRpc(grpcPort);
8388

‎ydb/core/sys_view/ut_common.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ struct TTestEnvSettings {
2929
bool EnableTableCacheModes = false;
3030
TMaybe<bool> EnableRealSystemViewPaths;
3131
NKikimrProto::TAuthConfig AuthConfig = {};
32+
TMaybe<ui32> DataShardStatsReportIntervalSeconds;
3233
};
3334

3435
class TTestEnv {

‎ydb/core/sys_view/ut_kqp.cpp‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2832,9 +2832,7 @@ R"(CREATE TABLE `test_show_create` (
28322832
}
28332833

28342834
Y_UNIT_TEST(PartitionStatsLocksFields) {
2835-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2836-
2837-
TTestEnv env;
2835+
TTestEnv env({.DataShardStatsReportIntervalSeconds = 0});
28382836
CreateRootTable(env, /* partitionCount */ 1, /* fillTable */ true);
28392837

28402838
TTableClient client(env.GetDriver());
@@ -2862,11 +2860,9 @@ R"(CREATE TABLE `test_show_create` (
28622860
}
28632861

28642862
Y_UNIT_TEST(PartitionStatsFields) {
2865-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2866-
28672863
auto nowUs = TInstant::Now().MicroSeconds();
28682864

2869-
TTestEnv env;
2865+
TTestEnv env({.DataShardStatsReportIntervalSeconds = 0});
28702866
CreateRootTable(env);
28712867
const auto describeResult = env.GetClient().Describe(env.GetServer().GetRuntime(), "Root/Table0");
28722868
const auto tablePathId = describeResult.GetPathId();
@@ -3357,11 +3353,9 @@ R"(CREATE TABLE `test_show_create` (
33573353
}
33583354

33593355
Y_UNIT_TEST(TopPartitionsByCpuFields) {
3360-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3361-
33623356
auto nowUs = TInstant::Now().MicroSeconds();
33633357

3364-
TTestEnv env(1, 4, {.EnableSVP = true});
3358+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
33653359
CreateTenantsAndTables(env);
33663360

33673361
TTableClient client(env.GetDriver());
@@ -3409,11 +3403,9 @@ R"(CREATE TABLE `test_show_create` (
34093403
}
34103404

34113405
Y_UNIT_TEST(TopPartitionsByCpuTables) {
3412-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3413-
34143406
constexpr ui64 partitionCount = 5;
34153407

3416-
TTestEnv env(1, 4, {.EnableSVP = true});
3408+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
34173409
CreateTenantsAndTables(env, true, partitionCount);
34183410

34193411
TTableClient client(env.GetDriver());
@@ -3439,11 +3431,9 @@ R"(CREATE TABLE `test_show_create` (
34393431
}
34403432

34413433
Y_UNIT_TEST(TopPartitionsByCpuRanges) {
3442-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3443-
34443434
constexpr ui64 partitionCount = 5;
34453435

3446-
TTestEnv env(1, 4, {.EnableSVP = true});
3436+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
34473437
CreateTenantsAndTables(env, true, partitionCount);
34483438

34493439
TTableClient client(env.GetDriver());
@@ -3520,11 +3510,13 @@ R"(CREATE TABLE `test_show_create` (
35203510
}
35213511

35223512
Y_UNIT_TEST(TopPartitionsByCpuFollowers) {
3523-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3524-
35253513
auto nowUs = TInstant::Now().MicroSeconds();
35263514

3527-
TTestEnv env(1, 4, {.EnableSVP = true, .EnableForceFollowers = true});
3515+
TTestEnv env(1, 4, {
3516+
.EnableSVP = true,
3517+
.EnableForceFollowers = true,
3518+
.DataShardStatsReportIntervalSeconds = 0,
3519+
});
35283520

35293521
auto& runtime = *env.GetServer().GetRuntime();
35303522
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
@@ -3695,9 +3687,7 @@ R"(CREATE TABLE `test_show_create` (
36953687
}
36963688

36973689
Y_UNIT_TEST(TopPartitionsByTliFields) {
3698-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3699-
3700-
TTestEnv env(1, 4, {.EnableSVP = true});
3690+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
37013691
CreateTenantsAndTables(env);
37023692

37033693
TTableClient client(env.GetDriver());

‎ydb/core/testlib/test_client.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -555,6 +555,7 @@ namespace Tests {
555555
appData.DataStreamsAuthFactory = Settings->DataStreamsAuthFactory.get();
556556
appData.PersQueueMirrorReaderFactory = Settings->PersQueueMirrorReaderFactory.get();
557557
appData.HiveConfig.MergeFrom(Settings->AppConfig->GetHiveConfig());
558+
appData.DataShardConfig.MergeFrom(Settings->AppConfig->GetDataShardConfig());
558559
appData.GraphConfig.MergeFrom(Settings->AppConfig->GetGraphConfig());
559560
appData.SqsConfig.MergeFrom(Settings->AppConfig->GetSqsConfig());
560561
appData.SharedCacheConfig.MergeFrom(Settings->AppConfig->GetSharedCacheConfig());

‎ydb/core/tx/datashard/datashard.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ using namespace NTabletFlatExecutor;
3535
// But in unit tests we want to test both scenarios
3636
bool gAllowLogBatchingDefaultValue = true;
3737

38-
TDuration gDbStatsReportInterval = TDuration::Seconds(10);
3938
ui64 gDbStatsDataSizeResolution = 10*1024*1024;
4039
ui64 gDbStatsRowCountResolution = 100000;
4140
ui32 gDbStatsHistogramBucketsCount = 10;

‎ydb/core/tx/datashard/datashard.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,6 @@ namespace NDataShard {
153153

154154
// NOTE: this switch should be modified only in tests !!!
155155
extern bool gAllowLogBatchingDefaultValue;
156-
extern TDuration gDbStatsReportInterval;
157156
extern ui64 gDbStatsDataSizeResolution;
158157
extern ui64 gDbStatsRowCountResolution;
159158
extern ui32 gDbStatsHistogramBucketsCount;

‎ydb/core/tx/datashard/datashard__stats.cpp‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -578,13 +578,19 @@ class TDataShard::TTxInitiateStatsUpdate : public NTabletFlatExecutor::TTransact
578578
}
579579
};
580580

581+
TDuration TDataShard::GetStatsReportInterval(const TAppData& appData) const {
582+
return TDuration::Seconds(
583+
appData.DataShardConfig.GetStatsReportIntervalSeconds());
584+
}
585+
581586
void TDataShard::UpdateTableStats(const TActorContext &ctx) {
582587
if (StatisticsDisabled)
583588
return;
584589

585-
TInstant now = AppData(ctx)->TimeProvider->Now();
590+
auto* appData = AppData(ctx);
591+
TInstant now = appData->TimeProvider->Now();
586592

587-
if (LastDbStatsUpdateTime + gDbStatsReportInterval > now)
593+
if (LastDbStatsUpdateTime + GetStatsReportInterval(*appData) > now)
588594
return;
589595

590596
if (State != TShardState::Ready && State != TShardState::Readonly)

‎ydb/core/tx/datashard/datashard_impl.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3348,13 +3348,16 @@ class TDataShard
33483348
NTabletPipe::SendData(ctx, StateReportPipe, ev.Release());
33493349
}
33503350

3351+
TDuration GetStatsReportInterval(const TAppData&) const;
3352+
33513353
void SendPeriodicTableStats(const TActorContext &ctx) {
33523354
if (StatisticsDisabled)
33533355
return;
33543356

3355-
TInstant now = AppData(ctx)->TimeProvider->Now();
3357+
auto* appData = AppData(ctx);
3358+
TInstant now = appData->TimeProvider->Now();
33563359

3357-
if (LastDbStatsReportTime + gDbStatsReportInterval > now)
3360+
if (LastDbStatsReportTime + GetStatsReportInterval(*appData) > now)
33583361
return;
33593362

33603363
auto* resourceMetrics = Executor()->GetResourceMetrics();

0 commit comments

Comments
 (0)