Skip to content

Commit 58e08de

Browse files
ztlpnztlpn
committed
Datashard configurable stats report intervals (#26862)
(cherry picked from commit efa0852)
1 parent 91daa20 commit 58e08de

File tree

23 files changed

+141
-64
lines changed

23 files changed

+141
-64
lines changed

‎ydb/core/client/flat_ut.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2627,7 +2627,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
26272627
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 1);
26282628

26292629
// Force stats reporting without delays
2630-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2630+
cleverServer.GetRuntime()->GetAppData()
2631+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
26312632

26322633
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
26332634
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_DEBUG);
@@ -2800,7 +2801,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
28002801
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 4);
28012802

28022803
// Force stats reporting without delays
2803-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2804+
cleverServer.GetRuntime()->GetAppData()
2805+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
28042806

28052807
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
28062808
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_DEBUG);
@@ -2906,7 +2908,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
29062908
TVector<ui64> initialPartitions = annoyingClient.GetTablePartitions("/dc-1/Dir/T1");
29072909

29082910
// Force stats reporting without delays
2909-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2911+
cleverServer.GetRuntime()->GetAppData()
2912+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
29102913
NDataShard::gDbStatsDataSizeResolution = 80000;
29112914

29122915
TString bigValue(100*1024, '0');

‎ydb/core/protos/datashard_config.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,6 @@ message TDataShardConfig {
2626
optional uint64 IncrementalRestoreReadAheadLo = 20 [default = 524288];
2727
optional uint64 IncrementalRestoreReadAheadHi = 21 [default = 1048576];
2828
optional uint64 InMemoryStateMigrationTimeoutMs = 24 [default = 1000];
29+
optional uint32 StatsReportIntervalSeconds = 25 [default = 10];
30+
optional uint32 BackupTableStatsReportIntervalSeconds = 26 [default = 30];
2931
}

‎ydb/core/sys_view/ut_common.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,11 @@ TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, const TTestEnvSettings&
7777

7878
Settings->AppConfig->MutableHiveConfig()->AddBalancerIgnoreTabletTypes(NKikimrTabletBase::TTabletTypes::SysViewProcessor);
7979

80+
if (settings.DataShardStatsReportIntervalSeconds) {
81+
Settings->AppConfig->MutableDataShardConfig()
82+
->SetStatsReportIntervalSeconds(*settings.DataShardStatsReportIntervalSeconds);
83+
}
84+
8085
Server = new Tests::TServer(*Settings);
8186
Server->EnableGRpc(grpcPort);
8287

‎ydb/core/sys_view/ut_common.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ struct TTestEnvSettings {
2828
bool EnableOlapCompression = false;
2929
TMaybe<bool> EnableRealSystemViewPaths;
3030
NKikimrProto::TAuthConfig AuthConfig = {};
31+
TMaybe<ui32> DataShardStatsReportIntervalSeconds;
3132
};
3233

3334
class TTestEnv {

‎ydb/core/sys_view/ut_kqp.cpp‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2704,9 +2704,7 @@ ALTER OBJECT `/Root/test_show_create` (TYPE TABLE) SET (ACTION = UPSERT_OPTIONS,
27042704
}
27052705

27062706
Y_UNIT_TEST(PartitionStatsLocksFields) {
2707-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2708-
2709-
TTestEnv env;
2707+
TTestEnv env({.DataShardStatsReportIntervalSeconds = 0});
27102708
CreateRootTable(env, /* partitionCount */ 1, /* fillTable */ true);
27112709

27122710
TTableClient client(env.GetDriver());
@@ -2734,11 +2732,9 @@ ALTER OBJECT `/Root/test_show_create` (TYPE TABLE) SET (ACTION = UPSERT_OPTIONS,
27342732
}
27352733

27362734
Y_UNIT_TEST(PartitionStatsFields) {
2737-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2738-
27392735
auto nowUs = TInstant::Now().MicroSeconds();
27402736

2741-
TTestEnv env;
2737+
TTestEnv env({.DataShardStatsReportIntervalSeconds = 0});
27422738
CreateRootTable(env);
27432739
const auto describeResult = env.GetClient().Describe(env.GetServer().GetRuntime(), "Root/Table0");
27442740
const auto tablePathId = describeResult.GetPathId();
@@ -3229,11 +3225,9 @@ ALTER OBJECT `/Root/test_show_create` (TYPE TABLE) SET (ACTION = UPSERT_OPTIONS,
32293225
}
32303226

32313227
Y_UNIT_TEST(TopPartitionsByCpuFields) {
3232-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3233-
32343228
auto nowUs = TInstant::Now().MicroSeconds();
32353229

3236-
TTestEnv env(1, 4, {.EnableSVP = true});
3230+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
32373231
CreateTenantsAndTables(env);
32383232

32393233
TTableClient client(env.GetDriver());
@@ -3281,11 +3275,9 @@ ALTER OBJECT `/Root/test_show_create` (TYPE TABLE) SET (ACTION = UPSERT_OPTIONS,
32813275
}
32823276

32833277
Y_UNIT_TEST(TopPartitionsByCpuTables) {
3284-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3285-
32863278
constexpr ui64 partitionCount = 5;
32873279

3288-
TTestEnv env(1, 4, {.EnableSVP = true});
3280+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
32893281
CreateTenantsAndTables(env, true, partitionCount);
32903282

32913283
TTableClient client(env.GetDriver());
@@ -3311,11 +3303,9 @@ ALTER OBJECT `/Root/test_show_create` (TYPE TABLE) SET (ACTION = UPSERT_OPTIONS,
33113303
}
33123304

33133305
Y_UNIT_TEST(TopPartitionsByCpuRanges) {
3314-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3315-
33163306
constexpr ui64 partitionCount = 5;
33173307

3318-
TTestEnv env(1, 4, {.EnableSVP = true});
3308+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
33193309
CreateTenantsAndTables(env, true, partitionCount);
33203310

33213311
TTableClient client(env.GetDriver());
@@ -3392,11 +3382,13 @@ ALTER OBJECT `/Root/test_show_create` (TYPE TABLE) SET (ACTION = UPSERT_OPTIONS,
33923382
}
33933383

33943384
Y_UNIT_TEST(TopPartitionsByCpuFollowers) {
3395-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3396-
33973385
auto nowUs = TInstant::Now().MicroSeconds();
33983386

3399-
TTestEnv env(1, 4, {.EnableSVP = true, .EnableForceFollowers = true});
3387+
TTestEnv env(1, 4, {
3388+
.EnableSVP = true,
3389+
.EnableForceFollowers = true,
3390+
.DataShardStatsReportIntervalSeconds = 0,
3391+
});
34003392

34013393
auto& runtime = *env.GetServer().GetRuntime();
34023394
runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE);
@@ -3567,9 +3559,7 @@ ALTER OBJECT `/Root/test_show_create` (TYPE TABLE) SET (ACTION = UPSERT_OPTIONS,
35673559
}
35683560

35693561
Y_UNIT_TEST(TopPartitionsByTliFields) {
3570-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3571-
3572-
TTestEnv env(1, 4, {.EnableSVP = true});
3562+
TTestEnv env(1, 4, {.EnableSVP = true, .DataShardStatsReportIntervalSeconds = 0});
35733563
CreateTenantsAndTables(env);
35743564

35753565
TTableClient client(env.GetDriver());

‎ydb/core/testlib/test_client.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -554,6 +554,7 @@ namespace Tests {
554554
appData.DataStreamsAuthFactory = Settings->DataStreamsAuthFactory.get();
555555
appData.PersQueueMirrorReaderFactory = Settings->PersQueueMirrorReaderFactory.get();
556556
appData.HiveConfig.MergeFrom(Settings->AppConfig->GetHiveConfig());
557+
appData.DataShardConfig.MergeFrom(Settings->AppConfig->GetDataShardConfig());
557558
appData.GraphConfig.MergeFrom(Settings->AppConfig->GetGraphConfig());
558559
appData.SqsConfig.MergeFrom(Settings->AppConfig->GetSqsConfig());
559560
appData.SharedCacheConfig.MergeFrom(Settings->AppConfig->GetSharedCacheConfig());

‎ydb/core/tx/datashard/datashard.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ using namespace NTabletFlatExecutor;
3535
// But in unit tests we want to test both scenarios
3636
bool gAllowLogBatchingDefaultValue = true;
3737

38-
TDuration gDbStatsReportInterval = TDuration::Seconds(10);
3938
ui64 gDbStatsDataSizeResolution = 10*1024*1024;
4039
ui64 gDbStatsRowCountResolution = 100000;
4140
ui32 gDbStatsHistogramBucketsCount = 10;

‎ydb/core/tx/datashard/datashard.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,6 @@ namespace NDataShard {
153153

154154
// NOTE: this switch should be modified only in tests !!!
155155
extern bool gAllowLogBatchingDefaultValue;
156-
extern TDuration gDbStatsReportInterval;
157156
extern ui64 gDbStatsDataSizeResolution;
158157
extern ui64 gDbStatsRowCountResolution;
159158
extern ui32 gDbStatsHistogramBucketsCount;

‎ydb/core/tx/datashard/datashard__stats.cpp‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -596,13 +596,31 @@ class TDataShard::TTxInitiateStatsUpdate : public NTabletFlatExecutor::TTransact
596596
}
597597
};
598598

599+
TDuration TDataShard::GetStatsReportInterval(const TAppData& appData) const {
600+
const auto& userTables = GetUserTables();
601+
const bool isBackup = !userTables.empty() && std::all_of(userTables.begin(), userTables.end(),
602+
[](const auto& kv) { return kv.second->IsBackup; });
603+
604+
if (isBackup) {
605+
// Clamp the interval for backup tables to the value for ordinary tables, as it
606+
// makes no sense for the latter to be longer than the former.
607+
auto interval = std::max(
608+
appData.DataShardConfig.GetBackupTableStatsReportIntervalSeconds(),
609+
appData.DataShardConfig.GetStatsReportIntervalSeconds());
610+
return TDuration::Seconds(interval);
611+
} else {
612+
return TDuration::Seconds(appData.DataShardConfig.GetStatsReportIntervalSeconds());
613+
}
614+
}
615+
599616
void TDataShard::UpdateTableStats(const TActorContext &ctx) {
600617
if (StatisticsDisabled)
601618
return;
602619

603-
TInstant now = AppData(ctx)->TimeProvider->Now();
620+
auto* appData = AppData(ctx);
621+
TInstant now = appData->TimeProvider->Now();
604622

605-
if (LastDbStatsUpdateTime + gDbStatsReportInterval > now)
623+
if (LastDbStatsUpdateTime + GetStatsReportInterval(*appData) > now)
606624
return;
607625

608626
if (State != TShardState::Ready && State != TShardState::Readonly)

‎ydb/core/tx/datashard/datashard_impl.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3334,13 +3334,16 @@ class TDataShard
33343334
NTabletPipe::SendData(ctx, StateReportPipe, ev.Release());
33353335
}
33363336

3337+
TDuration GetStatsReportInterval(const TAppData&) const;
3338+
33373339
void SendPeriodicTableStats(const TActorContext &ctx) {
33383340
if (StatisticsDisabled)
33393341
return;
33403342

3341-
TInstant now = AppData(ctx)->TimeProvider->Now();
3343+
auto* appData = AppData(ctx);
3344+
TInstant now = appData->TimeProvider->Now();
33423345

3343-
if (LastDbStatsReportTime + gDbStatsReportInterval > now)
3346+
if (LastDbStatsReportTime + GetStatsReportInterval(*appData) > now)
33443347
return;
33453348

33463349
auto* resourceMetrics = Executor()->GetResourceMetrics();

0 commit comments

Comments
 (0)