Issue 2725: add direct sending Acquire/ReleaseDevices from Volume to …

…DA (#2911) * issue-2725: add direct acruire release disk sending * issue-2725: correct issues * issue-2725: remove cashed acquire requests sending * issue-2725: mv check * issue-2725: add more tests * issue-2725: rename config flag * issue-2725: correct issues * issue-2725: refactor tests * issue-2725: change timeout * issue-2725: change test suite name and correct issues
ydb-platform · Jan 28, 2025 · 9096bf3 · 9096bf3
1 parent 8abac68
commit 9096bf3
Show file tree

Hide file tree

Showing 23 changed files with 1,322 additions and 21 deletions.
diff --git a/cloud/blockstore/config/storage.proto b/cloud/blockstore/config/storage.proto
@@ -1107,4 +1107,7 @@ message TStorageServiceConfig
     // When enabled, tag "use-intermediate-write-buffer" will be added
     // after scrubbing finds a mismatch
     optional bool AutomaticallyEnableBufferCopyingAfterChecksumMismatch = 403;
+
+    // Enabling direct sending AcquireDevices to disk agent.
+    optional bool NonReplicatedVolumeDirectAcquireEnabled = 404;
 }
diff --git a/cloud/blockstore/libs/storage/core/config.cpp b/cloud/blockstore/libs/storage/core/config.cpp
@@ -494,6 +494,7 @@ TDuration MSeconds(ui32 value)
     xxx(VolumeProxyCacheRetryDuration,             TDuration, Seconds(15)     )\
                                                                                \
     xxx(UseDirectCopyRange,                             bool,      false         )\
+    xxx(NonReplicatedVolumeDirectAcquireEnabled,        bool,      false         )\
     xxx(MaxShadowDiskFillBandwidth,                     ui32,      512           )\
     xxx(MaxShadowDiskFillIoDepth,                       ui32,      1             )\
     xxx(BackgroundOperationsTotalBandwidth,             ui32,      1024          )\

diff --git a/cloud/blockstore/libs/storage/core/config.h b/cloud/blockstore/libs/storage/core/config.h
@@ -633,6 +633,7 @@ class TStorageConfig
     bool GetYdbViewerServiceEnabled() const;
 
     bool GetAutomaticallyEnableBufferCopyingAfterChecksumMismatch() const;
+    [[nodiscard]] bool GetNonReplicatedVolumeDirectAcquireEnabled() const;
 };
 
 ui64 GetAllocationUnit(

diff --git a/cloud/blockstore/libs/storage/core/proto_helpers.cpp b/cloud/blockstore/libs/storage/core/proto_helpers.cpp
@@ -444,4 +444,15 @@ ui64 GetVolumeRequestId(const TEvDiskAgent::TEvZeroDeviceBlocksRequest& request)
     return request.Record.GetVolumeRequestId();
 }
 
+TString LogDevices(const TVector<NProto::TDeviceConfig>& devices)
+{
+    TStringBuilder sb;
+    sb << "( ";
+    for (const auto& d: devices) {
+        sb << d.GetDeviceUUID() << "@" << d.GetAgentId() << " ";
+    }
+    sb << ")";
+    return sb;
+}
+
 }   // namespace NCloud::NBlockStore::NStorage
diff --git a/cloud/blockstore/libs/storage/core/proto_helpers.h b/cloud/blockstore/libs/storage/core/proto_helpers.h
@@ -223,4 +223,5 @@ TBlockRange64 BuildRequestBlockRange(
 ui64 GetVolumeRequestId(const TEvDiskAgent::TEvWriteDeviceBlocksRequest& request);
 ui64 GetVolumeRequestId(const TEvDiskAgent::TEvZeroDeviceBlocksRequest& request);
 
+TString LogDevices(const TVector<NProto::TDeviceConfig>& devices);
 }   // namespace NCloud::NBlockStore::NStorage
diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.cpp
@@ -879,19 +879,6 @@ bool ToLogicalBlocks(NProto::TDeviceConfig& device, ui32 logicalBlockSize)
 
 ////////////////////////////////////////////////////////////////////////////////
 
-TString LogDevices(const TVector<NProto::TDeviceConfig>& devices)
-{
-    TStringBuilder sb;
-    sb << "( ";
-    for (const auto& d: devices) {
-        sb << d.GetDeviceUUID() << "@" << d.GetAgentId() << " ";
-    }
-    sb << ")";
-    return sb;
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
 void TDiskRegistryActor::OnDiskAcquired(
     TVector<TAgentAcquireDevicesCachedRequest> sentAcquireRequests)
 {
@@ -943,6 +930,9 @@ void TDiskRegistryActor::SendCachedAcquireRequestsToAgent(
     const TActorContext& ctx,
     const NProto::TAgentConfig& config)
 {
+    if (Config->GetNonReplicatedVolumeDirectAcquireEnabled()) {
+        return;
+    }
     auto& acquireCacheByAgentId = State->GetAcquireCacheByAgentId();
     auto cacheIt = acquireCacheByAgentId.find(config.GetAgentId());
     if (cacheIt == acquireCacheByAgentId.end()) {

diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.h b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor.h
@@ -515,6 +515,4 @@ class TDiskRegistryActor final
 TDiskRegistryStateSnapshot MakeNewLoadState(
     NProto::TDiskRegistryStateBackup&& backup);
 bool ToLogicalBlocks(NProto::TDeviceConfig& device, ui32 logicalBlockSize);
-TString LogDevices(const TVector<NProto::TDeviceConfig>& devices);
-
 }   // namespace NCloud::NBlockStore::NStorage
diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_acquire.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_acquire.cpp
@@ -1,6 +1,7 @@
 #include "disk_registry_actor.h"
 
 #include <cloud/blockstore/libs/storage/api/disk_agent.h>
+#include <cloud/blockstore/libs/storage/core/proto_helpers.h>
 
 #include <cloud/blockstore/libs/kikimr/events.h>
 

diff --git a/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_release.cpp b/cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_release.cpp
@@ -1,4 +1,5 @@
 #include "disk_registry_actor.h"
+#include <cloud/blockstore/libs/storage/core/proto_helpers.h>
 
 #include <cloud/blockstore/libs/storage/api/disk_agent.h>
 

diff --git a/cloud/blockstore/libs/storage/testlib/disk_agent_mock.h b/cloud/blockstore/libs/storage/testlib/disk_agent_mock.h
@@ -100,6 +100,8 @@ class TDiskAgentMock final
             HFunc(TEvDiskAgent::TEvZeroDeviceBlocksRequest, HandleZeroDeviceBlocks);
             HFunc(TEvDiskAgent::TEvChecksumDeviceBlocksRequest, HandleChecksumDeviceBlocks);
             HFunc(TEvDiskAgent::TEvDirectCopyBlocksRequest, HandleDirectCopyBlocks);
+            HFunc(TEvDiskAgent::TEvAcquireDevicesRequest, HandleAcquireDevicesRequest);
+            HFunc(TEvDiskAgent::TEvReleaseDevicesRequest, HandleReleaseDevicesRequest);
 
             default:
                 Y_ABORT("Unexpected event %x", ev->GetTypeRewrite());
@@ -315,6 +317,26 @@ class TDiskAgentMock final
     {
         State->CreateDirectCopyActorFunc(ev, ctx, SelfId());
     }
+
+    void HandleAcquireDevicesRequest(
+        TEvDiskAgent::TEvAcquireDevicesRequest::TPtr& ev,
+        const NActors::TActorContext& ctx)
+    {
+        auto response =
+            std::make_unique<TEvDiskAgent::TEvAcquireDevicesResponse>();
+
+        Reply(ctx, *ev, std::move(response));
+    }
+
+    void HandleReleaseDevicesRequest(
+        TEvDiskAgent::TEvReleaseDevicesRequest::TPtr& ev,
+        const NActors::TActorContext& ctx)
+    {
+        auto response =
+            std::make_unique<TEvDiskAgent::TEvReleaseDevicesResponse>();
+
+        Reply(ctx, *ev, std::move(response));
+    }
 };
 
 }   // namespace NCloud::NBlockStore::NStorage
diff --git a/cloud/blockstore/libs/storage/volume/ut/ya.make b/cloud/blockstore/libs/storage/volume/ut/ya.make
@@ -7,6 +7,7 @@ SRCS(
     volume_state_ut.cpp
     volume_ut.cpp
     volume_ut_checkpoint.cpp
+    volume_ut_session.cpp
     volume_ut_stats.cpp
 )
 

diff --git a/cloud/blockstore/libs/storage/volume/volume_actor.cpp b/cloud/blockstore/libs/storage/volume/volume_actor.cpp
@@ -1052,6 +1052,9 @@ STFUNC(TVolumeActor::StateWork)
         HFunc(
             TEvDiskRegistry::TEvAcquireDiskResponse,
             HandleAcquireDiskResponse);
+        HFunc(
+            TEvVolumePrivate::TEvDevicesAcquireFinished,
+            HandleDevicesAcquireFinished);
         HFunc(
             TEvVolumePrivate::TEvAcquireDiskIfNeeded,
             HandleAcquireDiskIfNeeded);
@@ -1060,6 +1063,9 @@ STFUNC(TVolumeActor::StateWork)
         HFunc(
             TEvDiskRegistry::TEvReleaseDiskResponse,
             HandleReleaseDiskResponse);
+        HFunc(
+            TEvVolumePrivate::TEvDevicesReleaseFinished,
+            HandleDevicesReleasedFinished);
         HFunc(
             TEvDiskRegistry::TEvAllocateDiskResponse,
             HandleAllocateDiskResponse);

diff --git a/cloud/blockstore/libs/storage/volume/volume_actor.h b/cloud/blockstore/libs/storage/volume/volume_actor.h
@@ -716,12 +716,26 @@ class TVolumeActor final
         const TEvDiskRegistry::TEvAcquireDiskResponse::TPtr& ev,
         const NActors::TActorContext& ctx);
 
+    void HandleDevicesAcquireFinishedImpl(
+        const NProto::TError& error,
+        const NActors::TActorContext& ctx);
+
     void AcquireDisk(
         const NActors::TActorContext& ctx,
         TString clientId,
         NProto::EVolumeAccessMode accessMode,
         ui64 mountSeqNumber);
 
+    void SendAcquireDevicesToAgents(
+        TString clientId,
+        NProto::EVolumeAccessMode accessMode,
+        ui64 mountSeqNumber,
+        const NActors::TActorContext& ctx);
+
+    void HandleDevicesAcquireFinished(
+        const TEvVolumePrivate::TEvDevicesAcquireFinished::TPtr& ev,
+        const NActors::TActorContext& ctx);
+
     void AcquireDiskIfNeeded(const NActors::TActorContext& ctx);
 
     void ScheduleAcquireDiskIfNeeded(const NActors::TActorContext& ctx);
@@ -742,8 +756,20 @@ class TVolumeActor final
         const TEvDiskRegistry::TEvReleaseDiskResponse::TPtr& ev,
         const NActors::TActorContext& ctx);
 
+    void HandleDevicesReleasedFinishedImpl(
+        const NProto::TError& error,
+        const NActors::TActorContext& ctx);
+
     void ReleaseDisk(const NActors::TActorContext& ctx, const TString& clientId);
 
+    void SendReleaseDevicesToAgents(
+        const TString& clientId,
+        const NActors::TActorContext& ctx);
+
+    void HandleDevicesReleasedFinished(
+        const TEvVolumePrivate::TEvDevicesReleaseFinished::TPtr& ev,
+        const NActors::TActorContext& ctx);
+
     void HandleAllocateDiskResponse(
         const TEvDiskRegistry::TEvAllocateDiskResponse::TPtr& ev,
         const NActors::TActorContext& ctx);