From a5df0d6f86d723d11823190d17683f80d8caed97 Mon Sep 17 00:00:00 2001 From: Egor Masharskii Date: Thu, 16 Jan 2025 01:32:15 +0000 Subject: [PATCH 1/3] initial commit --- cloud/filestore/libs/storage/api/service.h | 3 ++ .../libs/storage/service/service_actor.cpp | 14 +++++- .../libs/storage/service/service_actor.h | 10 ++++ .../service/service_actor_update_stats.cpp | 33 +++++++++++++ .../libs/storage/service/service_state.cpp | 2 + .../libs/storage/service/service_state.h | 4 ++ .../libs/storage/tablet/tablet_actor.cpp | 1 + .../tablet/tablet_actor_updateconfig.cpp | 2 + .../libs/storage/tablet/tablet_ut.cpp | 48 +++++++++++++++++++ .../libs/storage/testlib/service_client.h | 30 ++++++++++++ 10 files changed, 146 insertions(+), 1 deletion(-) diff --git a/cloud/filestore/libs/storage/api/service.h b/cloud/filestore/libs/storage/api/service.h index bdcaa8129cf..b738889bf18 100644 --- a/cloud/filestore/libs/storage/api/service.h +++ b/cloud/filestore/libs/storage/api/service.h @@ -87,16 +87,19 @@ struct TEvService const TString FileStoreId; const ui64 TabletId; const ui64 Generation; + const bool IsShard; NProtoPrivate::TFileSystemConfig Config; TRegisterLocalFileStore( TString fileStoreId, ui64 tablet, ui64 generation, + bool isShard, NProtoPrivate::TFileSystemConfig config) : FileStoreId(std::move(fileStoreId)) , TabletId(tablet) , Generation(generation) + , IsShard(isShard) , Config(std::move(config)) {} }; diff --git a/cloud/filestore/libs/storage/service/service_actor.cpp b/cloud/filestore/libs/storage/service/service_actor.cpp index 1b3105e3c06..c0bb6d292fb 100644 --- a/cloud/filestore/libs/storage/service/service_actor.cpp +++ b/cloud/filestore/libs/storage/service/service_actor.cpp @@ -52,7 +52,6 @@ void TStorageServiceActor::RegisterPages(const NActors::TActorContext& ctx) mon->RegisterActorPage(rootPage, "service", "Service", false, ctx.ExecutorThread.ActorSystem, SelfId()); } - } void TStorageServiceActor::RegisterCounters(const NActors::TActorContext& ctx) @@ -64,6 +63,18 @@ void TStorageServiceActor::RegisterCounters(const NActors::TActorContext& ctx) CpuWait = serverCounters->GetCounter("CpuWait", false); CpuWaitFailure = serverCounters->GetCounter("CpuWaitFailure", false); + + auto serviceCounters = rootGroup->GetSubgroup("component", "service"); + TotalFileSystemCount = serviceCounters->GetCounter("FileSystemCount", false); + TotalTabletCount = serviceCounters->GetCounter("TabletCount", false); + + auto hddCounters = serviceCounters->GetSubgroup("type", "hdd"); + HddFileSystemCount = hddCounters->GetCounter("FileSystemCount", false); + HddTabletCount = hddCounters->GetCounter("TabletCount", false); + + auto ssdCounters = serviceCounters->GetSubgroup("type", "ssd"); + SsdFileSystemCount = hddCounters->GetCounter("FileSystemCount", false); + SsdTabletCount = hddCounters->GetCounter("TabletCount", false); } void TStorageServiceActor::ScheduleUpdateStats(const NActors::TActorContext& ctx) @@ -166,6 +177,7 @@ void TStorageServiceActor::HandleRegisterLocalFileStore( msg->FileStoreId, msg->TabletId, msg->Generation, + msg->IsShard, std::move(msg->Config)); } } diff --git a/cloud/filestore/libs/storage/service/service_actor.h b/cloud/filestore/libs/storage/service/service_actor.h index b4db09abc7c..921f3010f95 100644 --- a/cloud/filestore/libs/storage/service/service_actor.h +++ b/cloud/filestore/libs/storage/service/service_actor.h @@ -46,6 +46,16 @@ class TStorageServiceActor final NMonitoring::TDynamicCounters::TCounterPtr CpuWait; NMonitoring::TDynamicCounters::TCounterPtr CpuWaitFailure; + + NMonitoring::TDynamicCounters::TCounterPtr TotalFileSystemCount; + NMonitoring::TDynamicCounters::TCounterPtr TotalTabletCount; + + NMonitoring::TDynamicCounters::TCounterPtr HddFileSystemCount; + NMonitoring::TDynamicCounters::TCounterPtr HddTabletCount; + + NMonitoring::TDynamicCounters::TCounterPtr SsdFileSystemCount; + NMonitoring::TDynamicCounters::TCounterPtr SsdTabletCount; + TInstant LastCpuWaitQuery; public: diff --git a/cloud/filestore/libs/storage/service/service_actor_update_stats.cpp b/cloud/filestore/libs/storage/service/service_actor_update_stats.cpp index c683a4718fd..a6167b75774 100644 --- a/cloud/filestore/libs/storage/service/service_actor_update_stats.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_update_stats.cpp @@ -22,6 +22,39 @@ void TStorageServiceActor::HandleUpdateStats( { Y_UNUSED(ev); + if (State) { + i64 totalFileSystems = 0; + i64 totalTablets = 0; + i64 hddFileSystems = 0; + i64 ssdFileSystems = 0; + i64 hddTablets = 0; + i64 ssdTablets = 0; + for (const auto& item: State->GetLocalFileStores()) { + constexpr auto MediaSsd = NProto::EStorageMediaKind::STORAGE_MEDIA_SSD; + if (!item.second.IsShard) { + auto& counter = + item.second.Config.GetStorageMediaKind() == MediaSsd ? + ssdFileSystems: + hddFileSystems; + ++counter; + ++totalFileSystems; + } + auto& counter = + item.second.Config.GetStorageMediaKind() == MediaSsd ? + ssdTablets: + hddTablets; + ++counter; + ++totalTablets; + + } + TotalFileSystemCount->Set(totalFileSystems); + TotalTabletCount->Set(totalTablets); + SsdFileSystemCount->Set(ssdFileSystems); + HddFileSystemCount->Set(hddFileSystems); + SsdTabletCount->Set(ssdTablets); + HddTabletCount->Set(hddTablets); + } + auto now = GetCycleCount(); for (auto it = InFlightRequests.begin(); it != InFlightRequests.end(); ) { const auto& request = it->second; diff --git a/cloud/filestore/libs/storage/service/service_state.cpp b/cloud/filestore/libs/storage/service/service_state.cpp index 4f2291f5998..78a2087084d 100644 --- a/cloud/filestore/libs/storage/service/service_state.cpp +++ b/cloud/filestore/libs/storage/service/service_state.cpp @@ -173,6 +173,7 @@ void TStorageServiceState::RegisterLocalFileStore( const TString& id, ui64 tablet, ui32 generation, + bool isShard, NProtoPrivate::TFileSystemConfig config) { // in case new instance registered before old unregistered or config was updated @@ -191,6 +192,7 @@ void TStorageServiceState::RegisterLocalFileStore( id, tablet, generation, + isShard, std::move(config))); } diff --git a/cloud/filestore/libs/storage/service/service_state.h b/cloud/filestore/libs/storage/service/service_state.h index eabd95295ff..764558d65a4 100644 --- a/cloud/filestore/libs/storage/service/service_state.h +++ b/cloud/filestore/libs/storage/service/service_state.h @@ -156,6 +156,7 @@ struct TLocalFileStore const TString FileStoreId; const ui64 TabletId; const ui32 Generation; + const bool IsShard; NProtoPrivate::TFileSystemConfig Config; @@ -163,10 +164,12 @@ struct TLocalFileStore TString id, ui64 tablet, ui32 generation, + bool isShard, NProtoPrivate::TFileSystemConfig config) : FileStoreId(std::move(id)) , TabletId(tablet) , Generation(generation) + , IsShard(isShard) , Config(std::move(config)) {} }; @@ -221,6 +224,7 @@ class TStorageServiceState const TString& id, ui64 tablet, ui32 generation, + bool isShard, NProtoPrivate::TFileSystemConfig config); void UnregisterLocalFileStore( const TString& id, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp index 7351c1a26a8..c06ad016294 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp @@ -1180,6 +1180,7 @@ void TIndexTabletActor::RegisterFileStore(const NActors::TActorContext& ctx) GetFileSystemId(), TabletID(), GetGeneration(), + GetFileSystem().GetShardNo() > 0, std::move(config)); ctx.Send(MakeStorageServiceId(), request.release()); diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp index 74bcaf74620..e163064553e 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp @@ -448,6 +448,8 @@ void TIndexTabletActor::CompleteTx_ConfigureAsShard( args.Request.GetShardNo(), JoinSeq(",", GetFileSystem().GetShardFileSystemIds()).c_str()); + RegisterFileStore(ctx); + auto response = std::make_unique(); diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut.cpp index 56dc860e86a..e3c1f80aafa 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut.cpp @@ -229,6 +229,54 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest) true, response->Record.GetStorageConfig().GetMultiTabletForwardingEnabled()); } + + Y_UNIT_TEST(ShouldNotifyServiceWhenFileSystemConfigChaged) + { + TTestEnv env; + env.CreateSubDomain("nfs"); + + const auto nodeIdx = env.CreateNode("nfs"); + const auto tabletId = env.BootIndexTablet(nodeIdx); + + TIndexTabletClient tablet(env.GetRuntime(), nodeIdx, tabletId); + + ui64 registerNonShardCount = 0; + ui64 registerShardCount = 0; + env.GetRuntime().SetEventFilter( + [&](auto& runtime, auto& event) + { + Y_UNUSED(runtime); + switch (event->GetTypeRewrite()) { + case TEvService::EvRegisterLocalFileStore: { + const auto* msg = event->template Get< + TEvService::TEvRegisterLocalFileStoreRequest>(); + if (tabletId != msg->TabletId) { + break; + } + if (msg->IsShard) { + ++registerShardCount; + } else { + ++registerNonShardCount; + } + } + } + return false; + }); + + tablet.UpdateConfig({ + .FileSystemId = "test_filesystem", + .CloudId = "test_cloud", + .FolderId = "test_folder", + }); + + UNIT_ASSERT_VALUES_EQUAL(1, registerNonShardCount); + UNIT_ASSERT_VALUES_EQUAL(0, registerShardCount); + + tablet.ConfigureAsShard(1); + + UNIT_ASSERT_VALUES_EQUAL(1, registerNonShardCount); + UNIT_ASSERT_VALUES_EQUAL(1, registerShardCount); + } } } // namespace NCloud::NFileStore::NStorage diff --git a/cloud/filestore/libs/storage/testlib/service_client.h b/cloud/filestore/libs/storage/testlib/service_client.h index fdda7e35713..b8214515f9b 100644 --- a/cloud/filestore/libs/storage/testlib/service_client.h +++ b/cloud/filestore/libs/storage/testlib/service_client.h @@ -69,6 +69,36 @@ class TServiceClient return Runtime; } + void RegisterLocalFileStore( + const TString& fileSystemId, + ui64 tabletId, + ui64 generation, + bool isShard, + NProtoPrivate::TFileSystemConfig config) + { + auto request = + std::make_unique( + fileSystemId, + tabletId, + generation, + isShard, + std::move(config)); + SendRequest(MakeStorageServiceId(), std::move(request)); + Runtime.DispatchEvents({}, TDuration::Seconds(1)); + } + + void UnregisterLocalFileStore( + const TString& fileSystemId, + ui64 generation) + { + auto request = + std::make_unique( + fileSystemId, + generation); + SendRequest(MakeStorageServiceId(), std::move(request)); + Runtime.DispatchEvents({}, TDuration::Seconds(1)); + } + THeaders InitSession( const TString& fileSystemId, const TString& clientId, From 0cd0039a2ed8ef09295734da39b46668d80fb383 Mon Sep 17 00:00:00 2001 From: Egor Masharskii Date: Thu, 16 Jan 2025 01:42:12 +0000 Subject: [PATCH 2/3] rebase --- .../libs/storage/service/service_ut.cpp | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/cloud/filestore/libs/storage/service/service_ut.cpp b/cloud/filestore/libs/storage/service/service_ut.cpp index 7abe7ec7a48..20546a8302f 100644 --- a/cloud/filestore/libs/storage/service/service_ut.cpp +++ b/cloud/filestore/libs/storage/service/service_ut.cpp @@ -3259,6 +3259,81 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest) CheckThreeStageWrites(NProto::STORAGE_MEDIA_SSD, true); CheckTwoStageReads(NProto::STORAGE_MEDIA_SSD, true); } + + Y_UNIT_TEST(ShouldUpdateFileSystemAndTabletCountersOnRegisterAndUnregister) + { + TTestEnv env; + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + + TServiceClient service(env.GetRuntime(), nodeIdx); + + service.RegisterLocalFileStore( + "test", + 1, // tablet id + 1, // generation + false, // isShard + {}); + + env.GetRuntime().AdvanceCurrentTime(TDuration::Seconds(15)); + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(1)); + + auto counters = env.GetRuntime().GetAppData(nodeIdx).Counters; + + auto fsCounter = counters + ->FindSubgroup("counters", "filestore") + ->FindSubgroup("component", "service") + ->GetCounter("FileSystemCount", false); + + auto hddFsCounter = counters + ->FindSubgroup("counters", "filestore") + ->FindSubgroup("component", "service") + ->FindSubgroup("type", "hdd") + ->GetCounter("FileSystemCount", false); + + auto ssdFsCounter = counters + ->FindSubgroup("counters", "filestore") + ->FindSubgroup("component", "service") + ->FindSubgroup("type", "ssd") + ->GetCounter("FileSystemCount", false); + + auto tabletCounter = counters + ->FindSubgroup("counters", "filestore") + ->FindSubgroup("component", "service") + ->GetCounter("TabletCount", false); + + auto hddTabletCounter = counters + ->FindSubgroup("counters", "filestore") + ->FindSubgroup("component", "service") + ->FindSubgroup("type", "hdd") + ->GetCounter("TabletCount", false); + + auto ssdTabletCounter = counters + ->FindSubgroup("counters", "filestore") + ->FindSubgroup("component", "service") + ->FindSubgroup("type", "ssd") + ->GetCounter("TabletCount", false); + + UNIT_ASSERT_VALUES_EQUAL(1, fsCounter->GetAtomic()); + UNIT_ASSERT_VALUES_EQUAL(1, tabletCounter->GetAtomic()); + UNIT_ASSERT_VALUES_EQUAL(1, hddFsCounter->GetAtomic()); + UNIT_ASSERT_VALUES_EQUAL(1, hddTabletCounter->GetAtomic()); + UNIT_ASSERT_VALUES_EQUAL(0, ssdFsCounter->GetAtomic()); + UNIT_ASSERT_VALUES_EQUAL(0, ssdTabletCounter->GetAtomic()); + + service.UnregisterLocalFileStore("test", 1); + + env.GetRuntime().AdvanceCurrentTime(TDuration::Seconds(15)); + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(1)); + + UNIT_ASSERT_VALUES_EQUAL(0, fsCounter->GetAtomic()); + UNIT_ASSERT_VALUES_EQUAL(0, tabletCounter->GetAtomic()); + UNIT_ASSERT_VALUES_EQUAL(0, hddFsCounter->GetAtomic()); + UNIT_ASSERT_VALUES_EQUAL(0, hddTabletCounter->GetAtomic()); + UNIT_ASSERT_VALUES_EQUAL(0, ssdFsCounter->GetAtomic()); + UNIT_ASSERT_VALUES_EQUAL(0, ssdTabletCounter->GetAtomic()); + } } } // namespace NCloud::NFileStore::NStorage From 28ce26452830da6d74ac732a7a4e3e8a16415ee3 Mon Sep 17 00:00:00 2001 From: Egor Masharskii Date: Fri, 17 Jan 2025 11:26:38 +0000 Subject: [PATCH 3/3] update --- cloud/filestore/libs/storage/tablet/tablet_ut.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut.cpp index e3c1f80aafa..cc23c5dce7b 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut.cpp @@ -230,7 +230,7 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest) response->Record.GetStorageConfig().GetMultiTabletForwardingEnabled()); } - Y_UNIT_TEST(ShouldNotifyServiceWhenFileSystemConfigChaged) + Y_UNIT_TEST(ShouldNotifyServiceWhenFileSystemConfigChanged) { TTestEnv env; env.CreateSubDomain("nfs");