Skip to content

Commit 2c30aa4

Browse files
ztlpnztlpn
committed
Datashard configurable stats report intervals (ydb-platform#26862)
(cherry picked from commit efa0852) Conflicts: ydb/core/protos/datashard_config.proto ydb/core/sys_view/ut_common.h ydb/core/sys_view/ut_kqp.cpp ydb/core/tx/schemeshard/ut_helpers/test_env.cpp ydb/core/tx/schemeshard/ut_helpers/test_env.h ydb/core/tx/schemeshard/ut_split_merge/ut_split_merge.cpp ydb/core/tx/schemeshard/ut_subdomain/ut_subdomain.cpp ydb/services/ydb/ydb_table_split_ut.cpp
1 parent f7bd7d3 commit 2c30aa4

File tree

23 files changed

+141
-57
lines changed

23 files changed

+141
-57
lines changed

‎ydb/core/client/flat_ut.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2950,7 +2950,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
29502950
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 1);
29512951

29522952
// Force stats reporting without delays
2953-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
2953+
cleverServer.GetRuntime()->GetAppData()
2954+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
29542955

29552956
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
29562957
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_DEBUG);
@@ -3123,7 +3124,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
31233124
UNIT_ASSERT_VALUES_EQUAL(partitions.size(), 4);
31243125

31253126
// Force stats reporting without delays
3126-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3127+
cleverServer.GetRuntime()->GetAppData()
3128+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
31273129

31283130
cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_DEBUG);
31293131
// cleverServer.GetRuntime()->SetLogPriority(NKikimrServices::TABLET_EXECUTOR, NActors::NLog::PRI_DEBUG);
@@ -3229,7 +3231,8 @@ Y_UNIT_TEST_SUITE(TFlatTest) {
32293231
TVector<ui64> initialPartitions = annoyingClient.GetTablePartitions("/dc-1/Dir/T1");
32303232

32313233
// Force stats reporting without delays
3232-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
3234+
cleverServer.GetRuntime()->GetAppData()
3235+
.DataShardConfig.SetStatsReportIntervalSeconds(0);
32333236
NDataShard::gDbStatsDataSizeResolution = 80000;
32343237

32353238
TString bigValue(100*1024, '0');

‎ydb/core/protos/datashard_config.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,6 @@ message TDataShardConfig {
2121
optional string CdcInitialScanTaskName = 17 [default = "cdc_initial_scan"];
2222
optional uint32 CdcInitialScanTaskPriority = 18 [default = 10];
2323
optional bool DisabledOnSchemeShard = 19 [default = false];
24+
optional uint32 StatsReportIntervalSeconds = 25 [default = 10];
25+
optional uint32 BackupTableStatsReportIntervalSeconds = 26 [default = 30];
2426
}

‎ydb/core/sys_view/ut_common.cpp‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,10 @@ NKikimrSubDomains::TSubDomainSettings GetSubDomainDefaultSettings(const TString
2626
return subdomain;
2727
}
2828

29-
TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools, ui32 pqTabletsN, bool enableSVP, bool disableSources) {
29+
TTestEnv::TTestEnv(
30+
ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools,
31+
ui32 pqTabletsN, bool enableSVP, bool disableSources,
32+
std::optional<ui32> dataShardStatsReportIntervalSeconds) {
3033
auto mbusPort = PortManager.GetPort();
3134
auto grpcPort = PortManager.GetPort();
3235

@@ -56,6 +59,11 @@ TTestEnv::TTestEnv(ui32 staticNodes, ui32 dynamicNodes, ui32 storagePools, ui32
5659
Settings->AppConfig->MutableTableServiceConfig()->SetEnableKqpDataQuerySourceRead(!disableSources);
5760
Settings->AppConfig->MutableHiveConfig()->AddBalancerIgnoreTabletTypes(NKikimrTabletBase::TTabletTypes::SysViewProcessor);
5861

62+
if (dataShardStatsReportIntervalSeconds) {
63+
Settings->AppConfig->MutableDataShardConfig()
64+
->SetStatsReportIntervalSeconds(*dataShardStatsReportIntervalSeconds);
65+
}
66+
5967
Server = new Tests::TServer(*Settings);
6068
Server->EnableGRpc(grpcPort);
6169

‎ydb/core/sys_view/ut_common.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ class TTestEnv {
2323

2424
public:
2525
TTestEnv(ui32 staticNodes = 1, ui32 dynamicNodes = 4, ui32 storagePools = 0,
26-
ui32 pqTabletsN = 0, bool enableSVP = false, bool disableSources = false);
26+
ui32 pqTabletsN = 0, bool enableSVP = false, bool disableSources = false,
27+
std::optional<ui32> dataShardStatsReportIntervalSeconds = std::nullopt);
2728

2829
template<typename... Args>
2930
TTestEnv(TDisableSourcesTag, Args&&... args)

‎ydb/core/sys_view/ut_kqp.cpp‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -717,11 +717,9 @@ Y_UNIT_TEST_SUITE(SystemView) {
717717
}
718718

719719
Y_UNIT_TEST(PartitionStatsFields) {
720-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
721-
722720
auto nowUs = TInstant::Now().MicroSeconds();
723721

724-
TTestEnv env;
722+
TTestEnv env(1, 4, 0, 0, false, false, 0);
725723
CreateRootTable(env);
726724

727725
TTableClient client(env.GetDriver());
@@ -1210,11 +1208,9 @@ Y_UNIT_TEST_SUITE(SystemView) {
12101208
}
12111209

12121210
Y_UNIT_TEST(TopPartitionsFields) {
1213-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
1214-
12151211
auto nowUs = TInstant::Now().MicroSeconds();
12161212

1217-
TTestEnv env(1, 4, 0, 0, true);
1213+
TTestEnv env(1, 4, 0, 0, true, false, 0);
12181214
CreateTenantsAndTables(env);
12191215

12201216
TTableClient client(env.GetDriver());
@@ -1262,11 +1258,9 @@ Y_UNIT_TEST_SUITE(SystemView) {
12621258
}
12631259

12641260
Y_UNIT_TEST(TopPartitionsTables) {
1265-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
1266-
12671261
constexpr ui64 partitionCount = 5;
12681262

1269-
TTestEnv env(1, 4, 0, 0, true);
1263+
TTestEnv env(1, 4, 0, 0, true, false, 0);
12701264
CreateTenantsAndTables(env, true, partitionCount);
12711265

12721266
TTableClient client(env.GetDriver());
@@ -1292,11 +1286,9 @@ Y_UNIT_TEST_SUITE(SystemView) {
12921286
}
12931287

12941288
Y_UNIT_TEST(TopPartitionsRanges) {
1295-
NDataShard::gDbStatsReportInterval = TDuration::Seconds(0);
1296-
12971289
constexpr ui64 partitionCount = 5;
12981290

1299-
TTestEnv env(1, 4, 0, 0, true);
1291+
TTestEnv env(1, 4, 0, 0, true, false, 0);
13001292
CreateTenantsAndTables(env, true, partitionCount);
13011293

13021294
TTableClient client(env.GetDriver());

‎ydb/core/testlib/test_client.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,7 @@ namespace Tests {
252252
appData.DataStreamsAuthFactory = Settings->DataStreamsAuthFactory.get();
253253
appData.PersQueueMirrorReaderFactory = Settings->PersQueueMirrorReaderFactory.get();
254254
appData.HiveConfig.MergeFrom(Settings->AppConfig->GetHiveConfig());
255+
appData.DataShardConfig.MergeFrom(Settings->AppConfig->GetDataShardConfig());
255256
appData.GraphConfig.MergeFrom(Settings->AppConfig->GetGraphConfig());
256257
appData.SqsConfig.MergeFrom(Settings->AppConfig->GetSqsConfig());
257258

‎ydb/core/tx/datashard/datashard.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ using namespace NTabletFlatExecutor;
3434
// But in unit tests we want to test both scenarios
3535
bool gAllowLogBatchingDefaultValue = true;
3636

37-
TDuration gDbStatsReportInterval = TDuration::Seconds(10);
3837
ui64 gDbStatsDataSizeResolution = 10*1024*1024;
3938
ui64 gDbStatsRowCountResolution = 100000;
4039
ui32 gDbStatsHistogramBucketsCount = 10;

‎ydb/core/tx/datashard/datashard.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,6 @@ namespace NDataShard {
148148

149149
// NOTE: this switch should be modified only in tests !!!
150150
extern bool gAllowLogBatchingDefaultValue;
151-
extern TDuration gDbStatsReportInterval;
152151
extern ui64 gDbStatsDataSizeResolution;
153152
extern ui64 gDbStatsRowCountResolution;
154153
extern ui32 gDbStatsHistogramBucketsCount;

‎ydb/core/tx/datashard/datashard__stats.cpp‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -597,13 +597,31 @@ class TDataShard::TTxInitiateStatsUpdate : public NTabletFlatExecutor::TTransact
597597
}
598598
};
599599

600+
TDuration TDataShard::GetStatsReportInterval(const TAppData& appData) const {
601+
const auto& userTables = GetUserTables();
602+
const bool isBackup = !userTables.empty() && std::all_of(userTables.begin(), userTables.end(),
603+
[](const auto& kv) { return kv.second->IsBackup; });
604+
605+
if (isBackup) {
606+
// Clamp the interval for backup tables to the value for ordinary tables, as it
607+
// makes no sense for the latter to be longer than the former.
608+
auto interval = std::max(
609+
appData.DataShardConfig.GetBackupTableStatsReportIntervalSeconds(),
610+
appData.DataShardConfig.GetStatsReportIntervalSeconds());
611+
return TDuration::Seconds(interval);
612+
} else {
613+
return TDuration::Seconds(appData.DataShardConfig.GetStatsReportIntervalSeconds());
614+
}
615+
}
616+
600617
void TDataShard::UpdateTableStats(const TActorContext &ctx) {
601618
if (StatisticsDisabled)
602619
return;
603620

604-
TInstant now = AppData(ctx)->TimeProvider->Now();
621+
auto* appData = AppData(ctx);
622+
TInstant now = appData->TimeProvider->Now();
605623

606-
if (LastDbStatsUpdateTime + gDbStatsReportInterval > now)
624+
if (LastDbStatsUpdateTime + GetStatsReportInterval(*appData) > now)
607625
return;
608626

609627
if (State != TShardState::Ready)

‎ydb/core/tx/datashard/datashard_impl.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3209,13 +3209,16 @@ class TDataShard
32093209
NTabletPipe::SendData(ctx, StateReportPipe, ev.Release());
32103210
}
32113211

3212+
TDuration GetStatsReportInterval(const TAppData&) const;
3213+
32123214
void SendPeriodicTableStats(const TActorContext &ctx) {
32133215
if (StatisticsDisabled)
32143216
return;
32153217

3216-
TInstant now = AppData(ctx)->TimeProvider->Now();
3218+
auto* appData = AppData(ctx);
3219+
TInstant now = appData->TimeProvider->Now();
32173220

3218-
if (LastDbStatsReportTime + gDbStatsReportInterval > now)
3221+
if (LastDbStatsReportTime + GetStatsReportInterval(*appData) > now)
32193222
return;
32203223

32213224
auto* resourceMetrics = Executor()->GetResourceMetrics();

0 commit comments

Comments
 (0)