Skip to content

Commit 194dbfa

Browse files
komarevtsev-dkomarevtsev-d
committed
Add parameter to server config to configure vmpte flush threshold
1 parent 732fb53 commit 194dbfa

File tree

15 files changed

+263
-42
lines changed

15 files changed

+263
-42
lines changed

‎cloud/blockstore/config/server.proto

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,13 +222,20 @@ message TServerConfig
222222
optional bool UseFakeRdmaClient = 120;
223223

224224
// Enable Discard/ZeroBlocks features for vhost devices
225-
optional bool VhostDiscardEnabled = 121;
225+
optional bool VhostDiscardEnabled = 121;
226226

227227
// Maximum request size in bytes for ZeroBlocks.
228228
// Requests exceeding this size will be split into smaller sub-requests,
229229
// each with a size up to MaxZeroBlocksSubRequestSize.
230230
// Max value for MaxZeroBlocksSubRequestSize is 2 GiB
231231
optional uint32 MaxZeroBlocksSubRequestSize = 122;
232+
233+
// If set to a non-zero value, PTEs of the external vhost server backing
234+
// the guest memory regions for blockdev are flushed (unmapped and mapped
235+
// back) every N bytes processed by the backend. E.g. if this value is 4096,
236+
// PTEs will be flushed after the guest reads/writes 8 512-byte blocks or
237+
// 1 4KiB block.
238+
optional uint64 VhostPteFlushByteThreshold = 123;
232239
}
233240

234241
////////////////////////////////////////////////////////////////////////////////

‎cloud/blockstore/libs/daemon/common/bootstrap.cpp

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -429,15 +429,13 @@ void TBootstrapBase::Init()
429429
&& !Configs->Options->TemporaryServer)
430430
{
431431
vhostEndpointListener = CreateExternalVhostEndpointListener(
432+
Configs->ServerConfig,
432433
Logging,
433434
ServerStats,
434435
Executor,
435-
Configs->ServerConfig->GetVhostServerPath(),
436436
Configs->Options->SkipDeviceLocalityValidation
437437
? TString {}
438438
: FQDNHostName(),
439-
Configs->ServerConfig->GetSocketAccessMode(),
440-
Configs->ServerConfig->GetVhostServerTimeoutAfterParentExit(),
441439
RdmaClient && RdmaClient->IsAlignedDataEnabled(),
442440
std::move(vhostEndpointListener));
443441

‎cloud/blockstore/libs/endpoints_vhost/external_vhost_server.cpp

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@
88
#include <cloud/blockstore/libs/diagnostics/server_stats.h>
99
#include <cloud/blockstore/libs/encryption/model/utils.h>
1010
#include <cloud/blockstore/libs/endpoints/endpoint_listener.h>
11+
#include <cloud/blockstore/libs/server/config.h>
1112
#include <cloud/blockstore/vhost-server/options.h>
13+
1214
#include <cloud/storage/core/libs/common/backoff_delay_provider.h>
1315
#include <cloud/storage/core/libs/common/media.h>
1416
#include <cloud/storage/core/libs/common/thread.h>
@@ -740,6 +742,7 @@ class TExternalVhostEndpointListener final
740742
, public IEndpointListener
741743
{
742744
private:
745+
const TServerAppConfigPtr ServerConfig;
743746
const ILoggingServicePtr Logging;
744747
const IServerStatsPtr ServerStats;
745748
const TExecutorPtr Executor;
@@ -757,25 +760,26 @@ class TExternalVhostEndpointListener final
757760

758761
public:
759762
TExternalVhostEndpointListener(
763+
TServerAppConfigPtr serverConfig,
760764
ILoggingServicePtr logging,
761765
IServerStatsPtr serverStats,
762766
TExecutorPtr executor,
763767
TString localAgentId,
764-
ui32 socketAccessMode,
765-
TDuration vhostServerTimeoutAfterParentExit,
766768
bool isAlignedDataEnabled,
767769
IEndpointListenerPtr fallbackListener,
768770
TExternalEndpointFactory endpointFactory)
769-
: Logging {std::move(logging)}
770-
, ServerStats {std::move(serverStats)}
771-
, Executor {std::move(executor)}
772-
, LocalAgentId {std::move(localAgentId)}
773-
, SocketAccessMode {socketAccessMode}
771+
: ServerConfig{std::move(serverConfig)}
772+
, Logging{std::move(logging)}
773+
, ServerStats{std::move(serverStats)}
774+
, Executor{std::move(executor)}
775+
, LocalAgentId{std::move(localAgentId)}
776+
, SocketAccessMode{ServerConfig->GetSocketAccessMode()}
774777
, IsAlignedDataEnabled(isAlignedDataEnabled)
775-
, FallbackListener {std::move(fallbackListener)}
776-
, EndpointFactory {std::move(endpointFactory)}
777-
, VhostServerTimeoutAfterParentExit{vhostServerTimeoutAfterParentExit}
778-
, Log {Logging->CreateLog("BLOCKSTORE_SERVER")}
778+
, FallbackListener{std::move(fallbackListener)}
779+
, EndpointFactory{std::move(endpointFactory)}
780+
, VhostServerTimeoutAfterParentExit{ServerConfig
781+
->GetVhostServerTimeoutAfterParentExit()}
782+
, Log{Logging->CreateLog("BLOCKSTORE_SERVER")}
779783
{
780784
FindRunningEndpoints();
781785
}
@@ -1066,6 +1070,13 @@ class TExternalVhostEndpointListener final
10661070
}
10671071
}
10681072

1073+
if (ui64 vhostPteFlushByteThreshold =
1074+
ServerConfig->GetVhostPteFlushByteThreshold())
1075+
{
1076+
args.emplace_back("--vmpte-flush-threshold");
1077+
args.emplace_back(ToString(vhostPteFlushByteThreshold));
1078+
}
1079+
10691080
for (const auto& device: volume.GetDevices()) {
10701081
const ui64 size = device.GetBlockCount() * volume.GetBlockSize();
10711082

@@ -1227,13 +1238,11 @@ class TExternalVhostEndpointListener final
12271238
////////////////////////////////////////////////////////////////////////////////
12281239

12291240
IEndpointListenerPtr CreateExternalVhostEndpointListener(
1241+
TServerAppConfigPtr serverConfig,
12301242
ILoggingServicePtr logging,
12311243
IServerStatsPtr serverStats,
12321244
TExecutorPtr executor,
1233-
TString binaryPath,
12341245
TString localAgentId,
1235-
ui32 socketAccessMode,
1236-
TDuration vhostServerTimeoutAfterParentExit,
12371246
bool isAlignedDataEnabled,
12381247
IEndpointListenerPtr fallbackListener)
12391248
{
@@ -1252,42 +1261,39 @@ IEndpointListenerPtr CreateExternalVhostEndpointListener(
12521261
.DiskId = diskId,
12531262
.ServerStats = serverStats
12541263
},
1255-
binaryPath,
1264+
serverConfig->GetVhostServerPath(),
12561265
std::move(args),
12571266
std::move(cgroups)
12581267
);
12591268
};
12601269

12611270
return std::make_shared<TExternalVhostEndpointListener>(
1271+
std::move(serverConfig),
12621272
std::move(logging),
12631273
std::move(serverStats),
12641274
std::move(executor),
12651275
std::move(localAgentId),
1266-
socketAccessMode,
1267-
vhostServerTimeoutAfterParentExit,
12681276
isAlignedDataEnabled,
12691277
std::move(fallbackListener),
12701278
std::move(defaultFactory));
12711279
}
12721280

12731281
IEndpointListenerPtr CreateExternalVhostEndpointListener(
1282+
TServerAppConfigPtr serverConfig,
12741283
ILoggingServicePtr logging,
12751284
IServerStatsPtr serverStats,
12761285
TExecutorPtr executor,
12771286
TString localAgentId,
1278-
ui32 socketAccessMode,
1279-
TDuration vhostServerTimeoutAfterParentExit,
12801287
bool isAlignedDataEnabled,
12811288
IEndpointListenerPtr fallbackListener,
12821289
TExternalEndpointFactory factory)
12831290
{
12841291
return std::make_shared<TExternalVhostEndpointListener>(
1292+
std::move(serverConfig),
12851293
std::move(logging),
12861294
std::move(serverStats),
12871295
std::move(executor),
12881296
std::move(localAgentId),
1289-
socketAccessMode,
1290-
vhostServerTimeoutAfterParentExit,
12911297
isAlignedDataEnabled,
12921298
std::move(fallbackListener),
12931299
std::move(factory));

‎cloud/blockstore/libs/endpoints_vhost/external_vhost_server.h

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@
44

55
#include <cloud/blockstore/libs/diagnostics/public.h>
66
#include <cloud/blockstore/libs/endpoints/public.h>
7+
#include <cloud/blockstore/libs/server/public.h>
78
#include <cloud/blockstore/libs/vhost/public.h>
8-
#include <cloud/storage/core/libs/coroutine/public.h>
99

10-
#include <cloud/storage/core/protos/error.pb.h>
10+
#include <cloud/storage/core/libs/common/error.h>
11+
#include <cloud/storage/core/libs/coroutine/public.h>
1112

1213
#include <library/cpp/threading/future/fwd.h>
1314

@@ -34,23 +35,20 @@ using TExternalEndpointFactory = std::function<IExternalEndpointPtr (
3435
////////////////////////////////////////////////////////////////////////////////
3536

3637
IEndpointListenerPtr CreateExternalVhostEndpointListener(
38+
TServerAppConfigPtr serverConfig,
3739
ILoggingServicePtr logging,
3840
IServerStatsPtr serverStats,
3941
TExecutorPtr executor,
40-
TString binaryPath,
4142
TString localAgentId,
42-
ui32 socketAccessMode,
43-
TDuration vhostServerTimeoutAfterParentExit,
4443
bool isAlignedDataEnabled,
4544
IEndpointListenerPtr fallbackListener);
4645

4746
IEndpointListenerPtr CreateExternalVhostEndpointListener(
47+
TServerAppConfigPtr serverConfig,
4848
ILoggingServicePtr logging,
4949
IServerStatsPtr serverStats,
5050
TExecutorPtr executor,
5151
TString localAgentId,
52-
ui32 socketAccessMode,
53-
TDuration vhostServerTimeoutAfterParentExit,
5452
bool isAlignedDataEnabled,
5553
IEndpointListenerPtr fallbackListener,
5654
TExternalEndpointFactory factory);

‎cloud/blockstore/libs/endpoints_vhost/external_vhost_server_ut.cpp

Lines changed: 96 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
#include <cloud/blockstore/libs/client/session_test.h>
44
#include <cloud/blockstore/libs/diagnostics/server_stats.h>
55
#include <cloud/blockstore/libs/endpoints/endpoint_listener.h>
6+
#include <cloud/blockstore/libs/server/config.h>
7+
8+
#include <cloud/storage/core/libs/common/random.h>
69
#include <cloud/storage/core/libs/coroutine/executor.h>
710
#include <cloud/storage/core/libs/diagnostics/logging.h>
811

@@ -257,18 +260,28 @@ struct TFixture
257260
THistory History;
258261

259262
IEndpointListenerPtr Listener =
260-
CreateEndpointListener(false); // no rdma aligned data
263+
CreateEndpointListener(false, {}); // no rdma aligned data
261264

262265
public:
263-
IEndpointListenerPtr CreateEndpointListener(bool isAlignedDataEnabled)
266+
IEndpointListenerPtr CreateEndpointListener(
267+
bool isAlignedDataEnabled,
268+
const NProto::TServerConfig& configPatch)
264269
{
270+
TServerAppConfigPtr serverConfigPtr;
271+
NProto::TServerAppConfig config;
272+
NProto::TServerConfig serverConfig;
273+
serverConfig.CopyFrom(configPatch);
274+
serverConfig.SetSocketAccessMode(S_IRGRP | S_IWGRP | S_IRUSR | S_IWUSR);
275+
serverConfig.SetVhostServerTimeoutAfterParentExit(
276+
TDuration::Seconds(30).MilliSeconds());
277+
*config.MutableServerConfig() = std::move(serverConfig);
278+
265279
return CreateExternalVhostEndpointListener(
280+
std::make_shared<TServerAppConfig>(config),
266281
Logging,
267282
ServerStats,
268283
Executor,
269284
LocalAgentId,
270-
S_IRGRP | S_IWGRP | S_IRUSR | S_IWUSR,
271-
TDuration::Seconds(30),
272285
isAlignedDataEnabled,
273286
CreateFallbackListener(),
274287
CreateExternalEndpointFactory());
@@ -400,7 +413,6 @@ Y_UNIT_TEST_SUITE(TExternalEndpointTest)
400413
Y_UNIT_TEST_F(ShouldStartAioExternalEndpoint, TFixture)
401414
{
402415
UNIT_ASSERT_VALUES_EQUAL(0, History.size());
403-
404416
{
405417
auto request = CreateDefaultStartEndpointRequest();
406418

@@ -681,7 +693,7 @@ Y_UNIT_TEST_SUITE(TExternalEndpointTest)
681693
}
682694

683695
{
684-
auto alignedDataListener = CreateEndpointListener(true);
696+
auto alignedDataListener = CreateEndpointListener(true, {});
685697

686698
auto request = CreateDefaultStartEndpointRequest();
687699

@@ -855,6 +867,84 @@ Y_UNIT_TEST_SUITE(TExternalEndpointTest)
855867
UNIT_ASSERT_C(stop, "actual entry: " << History[0].index());
856868
}
857869
}
870+
871+
Y_UNIT_TEST_F(ShoultPassVhostPteFlushByteThreshold, TFixture)
872+
{
873+
UNIT_ASSERT_VALUES_EQUAL(0, History.size());
874+
{
875+
const ui64 vhostPteFlushByteThreshold = RandInt<ui64>();
876+
NProto::TServerConfig serverConfig;
877+
serverConfig.SetVhostPteFlushByteThreshold(
878+
vhostPteFlushByteThreshold);
879+
Listener = CreateEndpointListener(false, serverConfig);
880+
auto request = CreateDefaultStartEndpointRequest();
881+
882+
auto error = Listener->StartEndpoint(request, Volume, Session)
883+
.GetValueSync();
884+
UNIT_ASSERT_C(!HasError(error), error);
885+
886+
UNIT_ASSERT_VALUES_EQUAL(3, History.size());
887+
888+
auto* create = std::get_if<TCreateExternalEndpoint>(&History[0]);
889+
UNIT_ASSERT_C(create, "actual entry: " << History[0].index());
890+
891+
UNIT_ASSERT_VALUES_EQUAL(Volume.GetDiskId(), create->DiskId);
892+
893+
/*
894+
--serial local0 2
895+
--disk-id vol0 2
896+
--block-size 512 2
897+
--socket-path /tmp/socket.vhost 2
898+
--socket-access-mode ... 2
899+
-q 2 2
900+
--device ... 2
901+
--device ... 2
902+
--read-only 1
903+
--wait-after-parent-exit ... 2
904+
--vmpte-flush-threshold ... 2
905+
19
906+
*/
907+
908+
UNIT_ASSERT_VALUES_EQUAL_C(
909+
21,
910+
create->CmdArgs.size(),
911+
JoinStrings(create->CmdArgs, " "));
912+
UNIT_ASSERT_VALUES_EQUAL("local0", GetArg(create->CmdArgs, "--serial"));
913+
UNIT_ASSERT_VALUES_EQUAL(
914+
"vol0",
915+
GetArg(create->CmdArgs, "--disk-id"));
916+
UNIT_ASSERT_VALUES_EQUAL(
917+
"512",
918+
GetArg(create->CmdArgs, "--block-size"));
919+
UNIT_ASSERT_VALUES_EQUAL(
920+
"30",
921+
GetArg(create->CmdArgs, "--wait-after-parent-exit"));
922+
UNIT_ASSERT_VALUES_EQUAL(
923+
ToString(vhostPteFlushByteThreshold),
924+
GetArg(create->CmdArgs, "--vmpte-flush-threshold"));
925+
926+
UNIT_ASSERT_VALUES_EQUAL(
927+
"/tmp/socket.vhost",
928+
GetArg(create->CmdArgs, "--socket-path"));
929+
930+
UNIT_ASSERT_VALUES_EQUAL("2", GetArg(create->CmdArgs, "-q"));
931+
UNIT_ASSERT(FindPtr(create->CmdArgs, "--read-only"));
932+
933+
auto devices = GetArgN(create->CmdArgs, "--device");
934+
UNIT_ASSERT_VALUES_EQUAL(2, devices.size());
935+
936+
History.clear();
937+
}
938+
939+
{
940+
auto error = Listener->StopEndpoint(SocketPath).GetValueSync();
941+
UNIT_ASSERT_C(!HasError(error), error);
942+
943+
UNIT_ASSERT_VALUES_EQUAL(1, History.size());
944+
auto* stop = std::get_if<TStopExternalEndpoint>(&History[0]);
945+
UNIT_ASSERT_C(stop, "actual entry: " << History[0].index());
946+
}
947+
}
858948
}
859949

860950
} // namespace NCloud::NBlockStore::NServer

‎cloud/blockstore/libs/endpoints_vhost/ya.make

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,12 @@ PEERDIR(
1111
cloud/blockstore/libs/common
1212
cloud/blockstore/libs/diagnostics
1313
cloud/blockstore/libs/endpoints
14+
cloud/blockstore/libs/server
1415
cloud/blockstore/libs/service
1516
cloud/blockstore/libs/vhost
1617

18+
cloud/storage/core/libs/common
19+
1720
library/cpp/getopt/small
1821
)
1922

‎cloud/blockstore/libs/server/config.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,8 @@ constexpr TDuration Seconds(int s)
106106
xxx(VhostServerTimeoutAfterParentExit, TDuration, Seconds(60) )\
107107
xxx(ChecksumFlags, NProto::TChecksumFlags, {} )\
108108
xxx(VhostDiscardEnabled, bool, false )\
109-
xxx(MaxZeroBlocksSubRequestSize, ui32, 0 )
109+
xxx(MaxZeroBlocksSubRequestSize, ui32, 0 )\
110+
xxx(VhostPteFlushByteThreshold, ui64, 0 )
110111
// BLOCKSTORE_SERVER_CONFIG
111112

112113
#define BLOCKSTORE_SERVER_DECLARE_CONFIG(name, type, value) \

‎cloud/blockstore/libs/server/config.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ class TServerAppConfig
139139
NProto::TChecksumFlags GetChecksumFlags() const;
140140
bool GetVhostDiscardEnabled() const;
141141
ui32 GetMaxZeroBlocksSubRequestSize() const;
142+
ui64 GetVhostPteFlushByteThreshold() const;
142143

143144
void Dump(IOutputStream& out) const override;
144145
void DumpHtml(IOutputStream& out) const override;

‎cloud/blockstore/vhost-server/backend_aio.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,8 @@ vhd_bdev_info TAioBackend::Init(const TOptions& options)
296296
.block_size = BlockSize,
297297
.num_queues = options.QueueCount, // Max count of virtio queues
298298
.total_blocks = totalBytes / BlockSize,
299-
.features = options.ReadOnly ? VHD_BDEV_F_READONLY : 0};
299+
.features = options.ReadOnly ? VHD_BDEV_F_READONLY : 0,
300+
.pte_flush_byte_threshold = options.PteFlushByteThreshold};
300301
}
301302

302303
void TAioBackend::Start()

0 commit comments

Comments
 (0)